From 6b5ffa60da1b970de39142a610a3b05ffb57739c Mon Sep 17 00:00:00 2001 From: Rob Royce Date: Thu, 8 Oct 2026 09:43:27 -0700 Subject: [PATCH 1/5] feat(storage): keep the inbox and browser preferences in the storage service - Schema 2: sources.project_id may be NULL. A source without a project is an inbox entry. The migration rebuilds the table in one transaction and keeps every row, position, and timestamp. - A transfer to a project is one PUT that sets projectId, so a repeated transfer cannot copy the source. PUT with "If-None-Match: *" creates a source only if the ID is new (412 otherwise). - Backup format version 2 includes inbox entries and their files. Version 1 backups still restore. Counts and restore results report the inbox. - GET/PUT /v1/preferences/: small JSON documents for client settings. Not part of a backup. - Session responses carry X-Knowledge-Session-TTL. - The legacy import reads "ingest-queue" and imports each inbox entry only if no source has its ID. Co-Authored-By: Claude Opus 5.5 --- src/kc_contracts/mapping.ts | 11 +- src/kc_contracts/storage.ts | 44 ++- src/kc_storage/src/backup.ts | 6 +- src/kc_storage/src/cli.ts | 1 + src/kc_storage/src/db.ts | 33 ++ src/kc_storage/src/errors.ts | 2 + src/kc_storage/src/http.ts | 25 ++ src/kc_storage/src/library.ts | 58 ++- src/kc_storage/src/migrate.ts | 46 ++- src/kc_storage/src/restore.ts | 13 +- src/kc_storage/src/validate.ts | 15 +- src/kc_storage/test/backup.test.ts | 5 +- src/kc_storage/test/browser-session.test.ts | 28 +- src/kc_storage/test/inbox.test.ts | 410 ++++++++++++++++++++ src/kc_storage/test/migration.test.ts | 92 ++++- src/kc_storage/test/restore.test.ts | 16 +- 16 files changed, 742 insertions(+), 63 deletions(-) create mode 100644 src/kc_storage/test/inbox.test.ts diff --git a/src/kc_contracts/mapping.ts b/src/kc_contracts/mapping.ts index a38e6e57..b2df4b8d 100644 --- a/src/kc_contracts/mapping.ts +++ b/src/kc_contracts/mapping.ts @@ -33,7 +33,8 @@ export interface MappedProject { export interface MappedSource { id: string; - projectId: string; + /** null: inbox */ + projectId: string | null; title: string; ingestType: string; assetId: string | null; @@ -72,7 +73,10 @@ export function projectToRecord(project: Json): MappedProject { }; } -export function sourceToRecord(source: Json, projectId: string): MappedSource { +export function sourceToRecord( + source: Json, + projectId: string | null +): MappedSource { return { id: source["id"]?.value, projectId, @@ -100,7 +104,8 @@ export function recordToSource(source: MappedSource): Json { id: { value: source.id }, title: source.title, ingestType: source.ingestType, - associatedProject: { value: source.projectId }, + // The UI uses an empty ID for a source without a project (inbox) + associatedProject: { value: source.projectId ?? "" }, }; if (source.assetId) { out["assetId"] = source.assetId; diff --git a/src/kc_contracts/storage.ts b/src/kc_contracts/storage.ts index 82d82993..71e41ef1 100644 --- a/src/kc_contracts/storage.ts +++ b/src/kc_contracts/storage.ts @@ -34,12 +34,13 @@ export interface ProjectRecord { export interface SourceRecord { id: string; - projectId: string; + /** null: the source is in the inbox, not in a project yet. */ + projectId: string | null; title: string; ingestType: string; /** Managed file of a file source. */ assetId: string | null; - /** Order inside the project. Set by the service. */ + /** Order inside the project or the inbox. Set by the service. */ position?: number; /** Every source field that has no column. */ data: JsonObject; @@ -71,6 +72,7 @@ export type StorageErrorCode = | "forbidden" | "not_found" | "conflict" + | "precondition_failed" | "payload_too_large" | "internal"; @@ -90,18 +92,46 @@ export interface SourceList { export const ASSET_FILENAME_HEADER = "x-knowledge-filename"; export const ASSET_ORIGINAL_PATH_HEADER = "x-knowledge-original-path"; +/** Library counts. `sources` includes the inbox entries. */ +export interface LibraryCounts { + projects: number; + sources: number; + inbox: number; + assets: number; +} + /** GET /v1/library */ export interface LibraryStatus { empty: boolean; - counts: { projects: number; sources: number; assets: number }; + counts: LibraryCounts; +} + +/** + * Request header for PUT /v1/sources/. With the value "*", the service + * creates the source only if no source has this ID. Otherwise it returns + * 412 and changes nothing. + */ +export const CREATE_ONLY_HEADER = "if-none-match"; + +/** GET and PUT /v1/preferences/. Not part of a library backup. */ +export interface PreferenceDocument { + key: string; + data: JsonObject; + updatedAt: string; } +/** + * Response header on requests that use a browser session: seconds until + * the session ends if no other request arrives. + */ +export const SESSION_TTL_HEADER = "x-knowledge-session-ttl"; + /** Result of POST /v1/restores: a validated backup that is not active yet. */ export interface RestorePreview { id: string; backupCreatedAt: string | null; backupVersion: number; - counts: { projects: number; sources: number; assets: number; bytes: number }; + counts: LibraryCounts & { bytes: number }; /** First 50 project names. */ projectNames: string[]; warnings: string[]; @@ -109,9 +139,13 @@ export interface RestorePreview { notIncluded: string[]; } +/** + * Version 1: projects, sources, and managed files. + * Version 2: also inbox entries (sources with projectId null). + */ export interface LibraryBackupManifest { format: "knowledge-library-backup"; - version: 1; + version: 1 | 2; createdAt: string; service: { version: string; schemaVersion: number }; projects: ProjectRecord[]; diff --git a/src/kc_storage/src/backup.ts b/src/kc_storage/src/backup.ts index d765fcf9..36a97f4d 100644 --- a/src/kc_storage/src/backup.ts +++ b/src/kc_storage/src/backup.ts @@ -1,7 +1,7 @@ /* * Portable library backup: a tar file with manifest.json and assets/. - * It holds projects, sources, and managed files. It does not hold chat - * history, UI preferences, settings, or credentials. + * It holds projects, sources, inbox entries, and managed files. It does + * not hold chat history, UI preferences, settings, or credentials. */ import type { Writable } from "node:stream"; @@ -17,7 +17,7 @@ export async function writeBackup(dir: DataDir, out: Writable) { const { library, assets } = dir; const manifest: LibraryBackupManifest = { format: BACKUP_FORMAT, - version: 1, + version: 2, createdAt: new Date().toISOString(), service: { version: VERSION, schemaVersion: SCHEMA_VERSION }, projects: library.listProjects(), diff --git a/src/kc_storage/src/cli.ts b/src/kc_storage/src/cli.ts index fae51fdc..19a39f68 100644 --- a/src/kc_storage/src/cli.ts +++ b/src/kc_storage/src/cli.ts @@ -151,6 +151,7 @@ function summary(report: Awaited>) { files: count(report.files), missingFiles: report.files.missing, superseded: report.superseded.length, + inboxEntries: report.inbox.length, rendererKeys: report.rendererKeys.length, }; } diff --git a/src/kc_storage/src/db.ts b/src/kc_storage/src/db.ts index 364aed6a..c7d18b92 100644 --- a/src/kc_storage/src/db.ts +++ b/src/kc_storage/src/db.ts @@ -58,6 +58,39 @@ const MIGRATIONS: Migration[] = [ ) STRICT; `, }, + { + // Inbox entries are sources with no project. Rebuild the table, because + // SQLite cannot remove NOT NULL from a column. No table refers to it. + version: 2, + sql: ` + CREATE TABLE sources_v2 ( + id TEXT PRIMARY KEY, + project_id TEXT REFERENCES projects (id) ON DELETE CASCADE, + title TEXT NOT NULL, + ingest_type TEXT NOT NULL, + asset_id TEXT REFERENCES assets (id) ON DELETE RESTRICT, + position INTEGER NOT NULL, + data TEXT NOT NULL CHECK (json_valid(data)), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) STRICT; + + INSERT INTO sources_v2 SELECT + id, project_id, title, ingest_type, asset_id, position, data, + created_at, updated_at + FROM sources; + + DROP TABLE sources; + ALTER TABLE sources_v2 RENAME TO sources; + CREATE INDEX sources_project ON sources (project_id, position); + + CREATE TABLE preferences ( + key TEXT PRIMARY KEY, + data TEXT NOT NULL CHECK (json_valid(data)), + updated_at TEXT NOT NULL + ) STRICT; + `, + }, ]; export const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version; diff --git a/src/kc_storage/src/errors.ts b/src/kc_storage/src/errors.ts index b4f33d59..cc48ef0d 100644 --- a/src/kc_storage/src/errors.ts +++ b/src/kc_storage/src/errors.ts @@ -22,5 +22,7 @@ export const notFound = (message: string) => new StorageError(404, "not_found", message); export const conflict = (message: string) => new StorageError(409, "conflict", message); +export const preconditionFailed = (message: string) => + new StorageError(412, "precondition_failed", message); export const tooLarge = (message: string) => new StorageError(413, "payload_too_large", message); diff --git a/src/kc_storage/src/http.ts b/src/kc_storage/src/http.ts index 7d810931..31c0d3ae 100644 --- a/src/kc_storage/src/http.ts +++ b/src/kc_storage/src/http.ts @@ -9,6 +9,8 @@ import type { AddressInfo } from "node:net"; import { ASSET_FILENAME_HEADER, ASSET_ORIGINAL_PATH_HEADER, + CREATE_ONLY_HEADER, + SESSION_TTL_HEADER, type ErrorResponse, type HealthResponse, type LibraryStatus, @@ -33,6 +35,7 @@ import { badRequest, forbidden, notFound, + preconditionFailed, StorageError, tooLarge, unauthorized, @@ -42,6 +45,7 @@ import { validId, validMediaType, validOriginalPath, + validPreference, validProject, validSource, } from "./validate.ts"; @@ -350,6 +354,10 @@ export function createServer(options: ServerOptions): http.Server { throw unauthorized("Missing or invalid bearer token or session."); } checkSessionRequest(req, method, session); + res.setHeader( + SESSION_TTL_HEADER, + String(Math.floor((sessions!.expiresAt(session) - Date.now()) / 1000)) + ); } if (resource === "projects" && !sub) { @@ -380,6 +388,10 @@ export function createServer(options: ServerOptions): http.Server { } if (method === "PUT" && id) { const record = validSource(id, await readJson(req, maxJson)); + const createOnly = req.headers[CREATE_ONLY_HEADER] === "*"; + if (createOnly && library.getSource(record.id)) { + throw preconditionFailed(`Source ${record.id} already exists.`); + } const result = library.putSource(record); return send(res, result === "created" ? 201 : 200, { source: library.getSource(record.id), @@ -448,6 +460,19 @@ export function createServer(options: ServerOptions): http.Server { } } + if (resource === "preferences" && id && !sub) { + const key = validId(id, "key"); + if (method === "GET") { + const doc = library.getPreference(key); + if (!doc) throw notFound(`Preference ${key} does not exist.`); + return send(res, 200, { preference: doc }); + } + if (method === "PUT") { + const data = validPreference(await readJson(req, maxJson)); + return send(res, 200, { preference: library.putPreference(key, data) }); + } + } + if (resource === "library" && !id && method === "GET") { const counts = library.counts(); const status: LibraryStatus = { diff --git a/src/kc_storage/src/library.ts b/src/kc_storage/src/library.ts index 369dfc1b..ca93527a 100644 --- a/src/kc_storage/src/library.ts +++ b/src/kc_storage/src/library.ts @@ -5,6 +5,9 @@ import type { DatabaseSync } from "node:sqlite"; import type { AssetRecord, + JsonObject, + LibraryCounts, + PreferenceDocument, ProjectRecord, SourceRecord, } from "../../kc_contracts/storage.ts"; @@ -24,7 +27,7 @@ interface ProjectRow { interface SourceRow { id: string; - project_id: string; + project_id: string | null; title: string; ingest_type: string; asset_id: string | null; @@ -130,16 +133,20 @@ export class Library { ); } - listSources(projectId?: string): SourceRecord[] { - const rows = (projectId + /** + * All sources (inbox first), the sources of one project, or with + * projectId null, the inbox. Each group is in position order. + */ + listSources(projectId?: string | null): SourceRecord[] { + const rows = (projectId === undefined ? this.db + .prepare("SELECT * FROM sources ORDER BY project_id, position, id") + .all() + : this.db .prepare( - "SELECT * FROM sources WHERE project_id = ? ORDER BY position, id" + "SELECT * FROM sources WHERE project_id IS ? ORDER BY position, id" ) - .all(projectId) - : this.db - .prepare("SELECT * FROM sources ORDER BY project_id, position, id") - .all()) as unknown as SourceRow[]; + .all(projectId)) as unknown as SourceRow[]; return rows.map(toSource); } @@ -148,8 +155,9 @@ export class Library { return row ? toSource(row as unknown as SourceRow) : undefined; } + /** A source with projectId null is an inbox entry. */ putSource(s: SourceRecord, createdAt?: string): WriteResult { - if (!this.getProject(s.projectId)) { + if (s.projectId !== null && !this.getProject(s.projectId)) { throw conflict(`Project ${s.projectId} does not exist.`); } if (s.assetId && !this.getAsset(s.assetId)) { @@ -161,7 +169,7 @@ export class Library { .prepare("SELECT * FROM sources WHERE id = ?") .get(s.id) as unknown as SourceRow | undefined; - // New sources and moved sources go to the end of their project + // New and moved sources go to the end of their project or the inbox const position = old && old.project_id === s.projectId ? old.position @@ -220,9 +228,9 @@ export class Library { ); } - private nextPosition(projectId: string): number { + private nextPosition(projectId: string | null): number { const row = this.db - .prepare("SELECT MAX(position) AS p FROM sources WHERE project_id = ?") + .prepare("SELECT MAX(position) AS p FROM sources WHERE project_id IS ?") .get(projectId) as { p: number | null }; return (row.p ?? -1) + 1; } @@ -280,14 +288,36 @@ export class Library { return new Set(rows.map((r) => r.id)); } - counts(): { projects: number; sources: number; assets: number } { + counts(): LibraryCounts { return this.db .prepare( `SELECT (SELECT COUNT(*) FROM projects) AS projects, (SELECT COUNT(*) FROM sources) AS sources, + (SELECT COUNT(*) FROM sources WHERE project_id IS NULL) AS inbox, (SELECT COUNT(*) FROM assets) AS assets` ) - .get() as { projects: number; sources: number; assets: number }; + .get() as unknown as LibraryCounts; + } + + getPreference(key: string): PreferenceDocument | undefined { + const row = this.db + .prepare("SELECT * FROM preferences WHERE key = ?") + .get(key) as + | { key: string; data: string; updated_at: string } + | undefined; + return row + ? { key: row.key, data: JSON.parse(row.data), updatedAt: row.updated_at } + : undefined; + } + + putPreference(key: string, data: JsonObject): PreferenceDocument { + this.db + .prepare( + `INSERT INTO preferences (key, data, updated_at) VALUES (?, ?, ?) + ON CONFLICT (key) DO UPDATE SET data = excluded.data, updated_at = excluded.updated_at` + ) + .run(key, JSON.stringify(data), now()); + return this.getPreference(key)!; } isEmpty(): boolean { diff --git a/src/kc_storage/src/migrate.ts b/src/kc_storage/src/migrate.ts index 827ad48b..2dfa99ba 100644 --- a/src/kc_storage/src/migrate.ts +++ b/src/kc_storage/src/migrate.ts @@ -67,6 +67,8 @@ export interface MigrationReport { superseded: { key: string; differentFields: string[] }[]; /** Derived source fields that were not stored, by field name. */ droppedDerivedFields: Record; + /** IDs of inbox entries (ingest-queue) planned as sources without a project. */ + inbox: string[]; /** Renderer keys that stay in localStorage (chat history, preferences). */ rendererKeys: string[]; rollback: { snapshot: string | null }; @@ -138,6 +140,7 @@ export async function migrate( files: { copied: [], reused: [], kept: [], missing: [] }, superseded: [], droppedDerivedFields: {}, + inbox: [], rendererKeys: [], rollback: { snapshot: null }, }; @@ -204,7 +207,7 @@ export async function migrate( } consumed.add("kc-projects"); - function planSource(ks: any, projectId: string) { + function planSource(ks: any, projectId: string | null) { const id = ks?.id?.value ?? "(no id)"; try { for (const field of DERIVED_SOURCE_FIELDS) { @@ -215,6 +218,10 @@ export async function migrate( } const mapped = sourceToRecord(ks, projectId); const record = validSource(mapped.id, mapped); + // A managed file that this library does not have: copy it by path again + if (record.assetId && !dir.library.getAsset(record.assetId)) { + record.assetId = null; + } if (plannedIds.has(record.id)) { report.sources.skipped.push({ id, @@ -267,6 +274,43 @@ export async function migrate( } } + // Inbox entries. Import an entry only if no source has its ID, so a + // repeated run or an entry already moved to a project stays unchanged. + const inboxRaw = data["ingest-queue"]; + if (inboxRaw !== undefined) { + consumed.add("ingest-queue"); + let queue: unknown; + try { + queue = JSON.parse(inboxRaw); + } catch { + queue = undefined; + } + if (!Array.isArray(queue)) { + report.sources.failed.push({ + id: "ingest-queue", + reason: "The inbox is not a JSON list.", + }); + } else { + for (const ks of queue) { + const id = ks?.id?.value; + if ( + typeof id === "string" && + (plannedIds.has(id) || dir.library.getSource(id)) + ) { + report.sources.skipped.push({ + id, + title: ks?.title, + reason: "The inbox entry is already in the library.", + }); + continue; + } + const before = planned.length; + planSource(ks, null); + if (planned.length > before) report.inbox.push(id); + } + } + } + report.rendererKeys = Object.keys(data) .filter((k) => !consumed.has(k)) .sort(); diff --git a/src/kc_storage/src/restore.ts b/src/kc_storage/src/restore.ts index 87fe842a..b7135a7f 100644 --- a/src/kc_storage/src/restore.ts +++ b/src/kc_storage/src/restore.ts @@ -31,7 +31,7 @@ import { readTarEntry, readTarIndex, streamTarEntry } from "./tar.ts"; import { validAssetId, validProject, validSource } from "./validate.ts"; import { BACKUP_FORMAT } from "./backup.ts"; -export const SUPPORTED_BACKUP_VERSIONS = [1]; +export const SUPPORTED_BACKUP_VERSIONS = [1, 2]; export const RESTORE_LIMITS = { archiveBytes: 8 * 1024 ** 3, @@ -43,7 +43,7 @@ export const RESTORE_LIMITS = { /** What a library backup never contains. Shown before confirmation. */ export const NOT_IN_LIBRARY_BACKUP = [ "Chat history", - "Inbox and UI preferences", + "UI preferences", "Application settings", "API keys", "Extracted-text cache", @@ -66,6 +66,7 @@ export interface StagedRestore { export interface RestoreResult { projects: number; sources: number; + inbox: number; assets: number; } @@ -234,7 +235,11 @@ export async function stageRestore( fail(`Invalid source ${JSON.stringify(s?.id)}: ${e.message}`); } unique(sourceIds, record.id, "source"); - if (!projectIds.has(record.projectId)) { + if (record.projectId === null) { + if (manifest.version < 2) { + fail(`Source ${record.id} has no project. Version 1 has no inbox.`); + } + } else if (!projectIds.has(record.projectId)) { fail( `Source ${record.id} refers to project ${record.projectId}, which is not in the backup.` ); @@ -305,6 +310,7 @@ export async function stageRestore( counts: { projects: projects.length, sources: sources.length, + inbox: sources.filter((s) => s.projectId === null).length, assets: assets.length, bytes, }, @@ -363,6 +369,7 @@ export function activateRestore( return { projects: staged.projects.length, sources: staged.sources.length, + inbox: staged.sources.filter((s) => s.projectId === null).length, assets: staged.assets.length, }; } catch (e) { diff --git a/src/kc_storage/src/validate.ts b/src/kc_storage/src/validate.ts index 7cd99cc1..970e5c02 100644 --- a/src/kc_storage/src/validate.ts +++ b/src/kc_storage/src/validate.ts @@ -109,9 +109,13 @@ export function validSource(pathId: string, input: unknown): SourceRecord { throw badRequest(`ingestType must be one of: ${INGEST_TYPES.join(", ")}.`); } const asset = body["assetId"]; + const project = body["projectId"]; + if (project === undefined) { + throw badRequest("projectId is required. Use null for the inbox."); + } return { id, - projectId: validId(body["projectId"], "projectId"), + projectId: project === null ? null : validId(project, "projectId"), title: text(body["title"] ?? "", "title", 4096, true), ingestType, assetId: asset === null || asset === undefined ? null : validAssetId(asset), @@ -119,6 +123,15 @@ export function validSource(pathId: string, input: unknown): SourceRecord { }; } +/** A preference document body: a JSON object up to 64 KiB. */ +export function validPreference(input: unknown): JsonObject { + const body = object(input, "body"); + if (JSON.stringify(body).length > 64 * 1024) { + throw badRequest("A preference document must be at most 64 KiB."); + } + return body; +} + /** A display filename. Path separators and control characters are removed. */ export function validFilename(raw: string | undefined): string { if (!raw) { diff --git a/src/kc_storage/test/backup.test.ts b/src/kc_storage/test/backup.test.ts index 152ac279..fc8c902d 100644 --- a/src/kc_storage/test/backup.test.ts +++ b/src/kc_storage/test/backup.test.ts @@ -99,6 +99,7 @@ test("backup restores into a different directory without the original files", as assert.deepEqual(JSON.parse(restored.stdout).restored, { projects: 2, sources: 3, + inbox: 0, assets: 2, }); @@ -175,8 +176,8 @@ test("the manifest holds no credentials and states its format", async () => { .toString() ); assert.equal(manifest.format, "knowledge-library-backup"); - assert.equal(manifest.version, 1); - assert.equal(manifest.service.schemaVersion, 1); + assert.equal(manifest.version, 2); + assert.equal(manifest.service.schemaVersion, 2); assert.deepEqual( manifest.assets.map((a: any) => a.path).sort(), manifest.assets.map((a: any) => `assets/${a.id}`).sort() diff --git a/src/kc_storage/test/browser-session.test.ts b/src/kc_storage/test/browser-session.test.ts index 45ebf5ad..418d920a 100644 --- a/src/kc_storage/test/browser-session.test.ts +++ b/src/kc_storage/test/browser-session.test.ts @@ -205,16 +205,20 @@ test("session requests need same-origin fetches, and writes need CSRF token and 401 ); - // Reads from the own page work; reads started by other sites do not - assert.equal( - ( - await rawRequest(svc, { - path: "/v1/projects", - headers: { Cookie: cookie, "Sec-Fetch-Site": "same-origin" }, - }) - ).status, - 200 - ); + // Reads from the own page work; reads started by other sites do not. + // Session responses report the remaining session time; bearer ones do not. + const read = await rawRequest(svc, { + path: "/v1/projects", + headers: { Cookie: cookie, "Sec-Fetch-Site": "same-origin" }, + }); + assert.equal(read.status, 200); + const ttl = Number(read.headers["x-knowledge-session-ttl"]); + assert.ok(ttl > 29 * 60 && ttl <= 30 * 60, `ttl ${ttl}`); + const bearer = await rawRequest(svc, { + path: "/v1/projects", + headers: { Authorization: `Bearer ${svc.token}` }, + }); + assert.equal(bearer.headers["x-knowledge-session-ttl"], undefined); for (const site of ["cross-site", "same-site"]) { assert.equal( ( @@ -407,7 +411,9 @@ test("managed files cannot run as the application; filenames are kept", async () assert.equal(res.headers["content-type"], c.contentType, c.type); assert.equal(res.headers["x-content-type-options"], "nosniff"); assert.equal( - String(res.headers["content-security-policy"] ?? "").startsWith("sandbox"), + String(res.headers["content-security-policy"] ?? "").startsWith( + "sandbox" + ), c.csp, c.type ); diff --git a/src/kc_storage/test/inbox.test.ts b/src/kc_storage/test/inbox.test.ts new file mode 100644 index 00000000..dcf8ac71 --- /dev/null +++ b/src/kc_storage/test/inbox.test.ts @@ -0,0 +1,410 @@ +import { afterEach, test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { readTarEntry, readTarIndex, TarWriter } from "../src/tar.ts"; +import { + FIXTURES, + cli, + client, + startService, + stopAll, + tempDir, + type Service, +} from "./helpers.ts"; + +afterEach(stopAll); + +const inboxEntry = (title: string, extra: object = {}) => ({ + projectId: null, + title, + ingestType: "website", + assetId: null, + data: { accessLink: `https://example.com/${title}`, topics: [title] }, + ...extra, +}); + +async function putCreateOnly(svc: Service, id: string, body: object) { + const res = await fetch(`${svc.url}/v1/sources/${id}`, { + method: "PUT", + headers: { + Authorization: `Bearer ${svc.token}`, + "Content-Type": "application/json", + "If-None-Match": "*", + }, + body: JSON.stringify(body), + }); + return { status: res.status, body: (await res.json()) as any }; +} + +const inboxOf = async (api: ReturnType) => + (await api.get("/v1/sources")).body.sources.filter( + (s: any) => s.projectId === null + ); + +test("inbox entries are sources without a project, in order, and survive a restart", async () => { + const dataDir = tempDir("inbox"); + let svc = await startService(dataDir); + let api = client(svc); + + for (const id of ["b", "a", "c"]) { + assert.equal( + (await api.put(`/v1/sources/in-${id}`, inboxEntry(id))).status, + 201 + ); + } + const pdf = ( + await api.upload( + path.join(FIXTURES, "recovery-fixture.pdf"), + "paper.pdf", + "application/pdf" + ) + ).body.asset; + await api.put( + "/v1/sources/in-file", + inboxEntry("paper.pdf", { ingestType: "file", assetId: pdf.id }) + ); + + // projectId is required; null means the inbox + const { projectId, ...noProject } = inboxEntry("x"); + assert.equal((await api.put("/v1/sources/in-x", noProject)).status, 400); + + assert.deepEqual((await api.get("/v1/library")).body.counts, { + projects: 0, + sources: 4, + inbox: 4, + assets: 1, + }); + + await svc.stop(); + svc = await startService(dataDir); + api = client(svc); + const inbox = await inboxOf(api); + assert.deepEqual( + inbox.map((s: any) => [s.id, s.position]), + [ + ["in-b", 0], + ["in-a", 1], + ["in-c", 2], + ["in-file", 3], + ] + ); + assert.deepEqual(inbox[0].data.topics, ["b"]); + assert.equal(inbox[3].assetId, pdf.id); + assert.ok( + (await api.content(pdf.id)).bytes.equals( + fs.readFileSync(path.join(FIXTURES, "recovery-fixture.pdf")) + ) + ); +}); + +test("a transfer moves the entry in one write; a repeated transfer adds nothing", async () => { + const svc = await startService(tempDir("transfer")); + const api = client(svc); + await api.put("/v1/projects/p1", { + name: "Project", + parentId: null, + data: {}, + }); + await api.put("/v1/sources/s-existing", { + ...inboxEntry("existing"), + projectId: "p1", + }); + await api.put("/v1/sources/in-1", inboxEntry("one")); + await api.put("/v1/sources/in-2", inboxEntry("two")); + + const moved = { ...inboxEntry("one"), projectId: "p1" }; + const first = await api.put("/v1/sources/in-1", moved); + assert.equal(first.status, 200); + assert.equal(first.body.source.projectId, "p1"); + assert.equal(first.body.source.position, 1); + + // Repeat after a lost response: same record, same position, no copy + const again = await api.put("/v1/sources/in-1", moved); + assert.equal(again.status, 200); + assert.equal(again.body.source.position, 1); + const inProject = (await api.get("/v1/sources?projectId=p1")).body.sources; + assert.deepEqual( + inProject.map((s: any) => s.id), + ["s-existing", "in-1"] + ); + assert.deepEqual( + (await inboxOf(api)).map((s: any) => s.id), + ["in-2"] + ); + assert.equal((await api.get("/v1/library")).body.counts.sources, 3); + + // A transfer to a project that does not exist changes nothing + const bad = await api.put("/v1/sources/in-2", { + ...inboxEntry("two"), + projectId: "missing", + }); + assert.equal(bad.status, 409); + assert.deepEqual( + (await inboxOf(api)).map((s: any) => s.id), + ["in-2"] + ); +}); + +test("a create-only write never moves or changes an existing source", async () => { + const svc = await startService(tempDir("create-only")); + const api = client(svc); + await api.put("/v1/projects/p1", { + name: "Project", + parentId: null, + data: {}, + }); + await api.put("/v1/sources/s1", { + ...inboxEntry("in project"), + projectId: "p1", + }); + + const blocked = await putCreateOnly( + svc, + "s1", + inboxEntry("stale inbox copy") + ); + assert.equal(blocked.status, 412); + assert.equal(blocked.body.error.code, "precondition_failed"); + const s1 = (await api.get("/v1/sources?projectId=p1")).body.sources[0]; + assert.equal(s1.title, "in project"); + + assert.equal( + (await putCreateOnly(svc, "new-1", inboxEntry("new"))).status, + 201 + ); + assert.equal( + (await putCreateOnly(svc, "new-1", inboxEntry("new"))).status, + 412 + ); + assert.deepEqual( + (await inboxOf(api)).map((s: any) => s.id), + ["new-1"] + ); +}); + +test("backup and restore keep inbox entries, their order, and their files", async () => { + const work = tempDir("inbox-bk-work"); + const source = tempDir("inbox-bk-source"); + const svc = await startService(source); + const api = client(svc); + await api.put("/v1/projects/p1", { + name: "Project", + parentId: null, + data: {}, + }); + const txt = ( + await api.upload( + path.join(FIXTURES, "recovery-note.txt"), + "note.txt", + "text/plain" + ) + ).body.asset; + await api.put( + "/v1/sources/in-file", + inboxEntry("note.txt", { ingestType: "file", assetId: txt.id }) + ); + await api.put("/v1/sources/in-web", inboxEntry("web")); + await api.put("/v1/sources/s1", { + ...inboxEntry("project source"), + projectId: "p1", + }); + const expected = (await api.get("/v1/sources")).body.sources.map( + ({ updatedAt, ...r }: any) => r + ); + await svc.stop(); + + const tar = path.join(work, "library.tar"); + assert.equal(cli(["backup", "--data-dir", source, "--out", tar]).status, 0); + const index = readTarIndex(tar, { allowName: () => true, maxEntries: 10 }); + const manifest = JSON.parse( + readTarEntry(tar, index.get("manifest.json")!).toString() + ); + assert.equal(manifest.version, 2); + + // Restore through the API: the preview counts the inbox + const target = tempDir("inbox-bk-target"); + const svc2 = await startService(target); + const res = await fetch(`${svc2.url}/v1/restores`, { + method: "POST", + headers: { + Authorization: `Bearer ${svc2.token}`, + "Content-Type": "application/x-tar", + }, + body: fs.readFileSync(tar), + }); + const { restore } = (await res.json()) as any; + assert.equal(restore.counts.inbox, 2); + const done = await fetch(`${svc2.url}/v1/restores/${restore.id}/activate`, { + method: "POST", + headers: { Authorization: `Bearer ${svc2.token}` }, + }); + assert.deepEqual(((await done.json()) as any).restored, { + projects: 1, + sources: 3, + inbox: 2, + assets: 1, + }); + + const api2 = client(svc2); + const restored = (await api2.get("/v1/sources")).body.sources.map( + ({ updatedAt, ...r }: any) => r + ); + assert.deepEqual(restored, expected); + assert.equal( + (await api2.content(txt.id)).bytes.toString(), + fs.readFileSync(path.join(FIXTURES, "recovery-note.txt"), "utf8") + ); +}); + +test("version 1 backups still restore; a version 1 backup cannot hold inbox entries", async () => { + const work = tempDir("v1-work"); + const source = tempDir("v1-source"); + const svc = await startService(source); + const api = client(svc); + await api.put("/v1/projects/p1", { + name: "Old Project", + parentId: null, + data: {}, + }); + await api.put("/v1/sources/s1", { ...inboxEntry("old"), projectId: "p1" }); + await svc.stop(); + const tar = path.join(work, "v2.tar"); + assert.equal(cli(["backup", "--data-dir", source, "--out", tar]).status, 0); + + const rewrite = async (out: string, change: (m: any) => void) => { + const index = readTarIndex(tar, { allowName: () => true, maxEntries: 10 }); + const manifest = JSON.parse( + readTarEntry(tar, index.get("manifest.json")!).toString() + ); + change(manifest); + const stream = fs.createWriteStream(out); + const writer = new TarWriter(stream); + await writer.addBuffer( + "manifest.json", + Buffer.from(JSON.stringify(manifest)) + ); + await writer.finish(); + await new Promise((r) => stream.end(r)); + return out; + }; + + const v1 = await rewrite(path.join(work, "v1.tar"), (m) => (m.version = 1)); + const ok = cli(["restore", "--data-dir", tempDir("v1-target"), "--from", v1]); + assert.equal(ok.status, 0, ok.stderr); + assert.deepEqual(JSON.parse(ok.stdout).restored, { + projects: 1, + sources: 1, + inbox: 0, + assets: 0, + }); + + const bad = await rewrite(path.join(work, "v1-inbox.tar"), (m) => { + m.version = 1; + m.sources[0].projectId = null; + }); + const refused = cli([ + "restore", + "--data-dir", + tempDir("v1-bad"), + "--from", + bad, + ]); + assert.notEqual(refused.status, 0); + assert.match(refused.stderr + refused.stdout, /Version 1 has no inbox/); +}); + +test("a schema 1 library upgrades to schema 2 without data changes", async () => { + const dataDir = tempDir("schema1"); + const db = new DatabaseSync(path.join(dataDir, "library.sqlite")); + db.exec(` + CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL) STRICT; + INSERT INTO schema_migrations VALUES (1, '2026-10-01T00:00:00.000Z'); + CREATE TABLE projects (id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL, + data TEXT NOT NULL CHECK (json_valid(data)), created_at TEXT NOT NULL, updated_at TEXT NOT NULL) STRICT; + CREATE TABLE assets (id TEXT PRIMARY KEY, sha256 TEXT NOT NULL, size INTEGER NOT NULL, media_type TEXT NOT NULL, + filename TEXT NOT NULL, original_path TEXT, created_at TEXT NOT NULL) STRICT; + CREATE INDEX assets_sha256 ON assets (sha256); + CREATE TABLE sources (id TEXT PRIMARY KEY, project_id TEXT NOT NULL REFERENCES projects (id) ON DELETE CASCADE, + title TEXT NOT NULL, ingest_type TEXT NOT NULL, asset_id TEXT REFERENCES assets (id) ON DELETE RESTRICT, + position INTEGER NOT NULL, data TEXT NOT NULL CHECK (json_valid(data)), created_at TEXT NOT NULL, updated_at TEXT NOT NULL) STRICT; + CREATE INDEX sources_project ON sources (project_id, position); + CREATE TABLE import_runs (id TEXT PRIMARY KEY, input_sha256 TEXT NOT NULL, started_at TEXT NOT NULL, + finished_at TEXT, status TEXT NOT NULL, report TEXT) STRICT; + INSERT INTO projects VALUES ('p1', NULL, 'Kept', '{"k":1}', '2026-10-01T00:00:00.000Z', '2026-10-01T00:00:00.000Z'); + INSERT INTO sources VALUES ('s1', 'p1', 'One', 'website', NULL, 0, '{"a":1}', '2026-10-01T00:00:00.000Z', '2026-10-01T00:00:00.000Z'); + INSERT INTO sources VALUES ('s2', 'p1', 'Two', 'note', NULL, 1, '{"b":2}', '2026-10-01T00:00:00.000Z', '2026-10-01T00:00:00.000Z'); + `); + db.close(); + + const svc = await startService(dataDir); + const api = client(svc); + assert.equal((await api.get("/health")).body.schemaVersion, 2); + const sources = (await api.get("/v1/sources")).body.sources; + assert.deepEqual( + sources.map((s: any) => [ + s.id, + s.projectId, + s.position, + s.data, + s.createdAt, + ]), + [ + ["s1", "p1", 0, { a: 1 }, "2026-10-01T00:00:00.000Z"], + ["s2", "p1", 1, { b: 2 }, "2026-10-01T00:00:00.000Z"], + ] + ); + assert.equal( + (await api.put("/v1/sources/in-1", inboxEntry("new"))).status, + 201 + ); + + // Foreign keys still hold after the rebuild + assert.equal((await api.del("/v1/projects/p1")).status, 204); + assert.deepEqual( + (await api.get("/v1/sources")).body.sources.map((s: any) => s.id), + ["in-1"] + ); +}); + +test("preferences persist across restarts and are not in the backup", async () => { + const dataDir = tempDir("prefs"); + let svc = await startService(dataDir); + let api = client(svc); + assert.equal((await api.get("/v1/preferences/browser-settings")).status, 404); + assert.equal( + (await api.put("/v1/preferences/browser-settings", ["x"])).status, + 400 + ); + assert.equal((await api.put("/v1/preferences/bad%20key", {})).status, 400); + assert.equal( + ( + await api.put("/v1/preferences/browser-settings", { + big: "x".repeat(70 * 1024), + }) + ).status, + 400 + ); + const put = await api.put("/v1/preferences/browser-settings", { + display: { theme: "dark" }, + }); + assert.equal(put.status, 200); + await svc.stop(); + + svc = await startService(dataDir); + api = client(svc); + const got = (await api.get("/v1/preferences/browser-settings")).body + .preference; + assert.deepEqual(got.data, { display: { theme: "dark" } }); + assert.equal((await api.get("/v1/library")).body.empty, true); + await svc.stop(); + + const tar = path.join(tempDir("prefs-bk"), "b.tar"); + assert.equal(cli(["backup", "--data-dir", dataDir, "--out", tar]).status, 0); + const index = readTarIndex(tar, { allowName: () => true, maxEntries: 10 }); + assert.doesNotMatch( + readTarEntry(tar, index.get("manifest.json")!).toString(), + /theme/ + ); +}); diff --git a/src/kc_storage/test/migration.test.ts b/src/kc_storage/test/migration.test.ts index 58a321c9..2bc5c203 100644 --- a/src/kc_storage/test/migration.test.ts +++ b/src/kc_storage/test/migration.test.ts @@ -20,13 +20,17 @@ const P1 = "11111111-1111-4111-8111-111111111111"; /** * A renderer backup with the record shapes found in real profiles: * embedded sources, a differing ks- copy, a standalone ks- record, - * an orphan ks- record, a missing file, derived fields, and other keys. + * an orphan ks- record, a missing file, derived fields, an inbox with + * a file and a website (plus an ID that is also in a project), and other + * keys. */ function legacyBackup(work: string) { const pdf = path.join(work, "paper.pdf"); const txt = path.join(work, "note.txt"); + const pending = path.join(work, "pending.txt"); fs.copyFileSync(path.join(FIXTURES, "recovery-fixture.pdf"), pdf); fs.copyFileSync(path.join(FIXTURES, "recovery-note.txt"), txt); + fs.writeFileSync(pending, "pending inbox file"); const gone = path.join(work, "deleted.pdf"); const fileSource = ( @@ -111,7 +115,24 @@ function legacyBackup(work: string) { // UI preference whose key looks like a source key "ks-table-rows": "10", theme: JSON.stringify({ name: "dark" }), - "ingest-queue": "[]", + "ingest-queue": JSON.stringify([ + { + ...fileSource("inbox-file", "pending.txt", pending, "text/plain", { + icon: undefined, + thumbnail: undefined, + }), + associatedProject: undefined, + }, + { + id: { value: "inbox-web" }, + title: "Pending Site", + ingestType: "website", + accessLink: "https://example.org/", + topics: ["later"], + }, + // Already in a project: the project copy wins + { ...sources[0], title: "Inbox copy of web-1" }, + ]), "current-project": P1, }; @@ -125,7 +146,7 @@ function legacyBackup(work: string) { data, }) ); - return { file, pdf, txt }; + return { file, pdf, txt, pending }; } function migrate(dataDir: string, input: string, env = {}) { @@ -179,15 +200,18 @@ test("migration imports, reports, and preserves data", async () => { assert.deepEqual(r.sources.imported.map((i: any) => i.id).sort(), [ "extra-1", "gone-1", + "inbox-file", + "inbox-web", "pdf-1", "txt-1", "web-1", ]); assert.deepEqual( r.sources.skipped.map((i: any) => i.id), - ["orphan-1"] + ["orphan-1", "web-1"] ); - assert.equal(r.files.copied.length, 2); + assert.deepEqual(r.inbox, ["inbox-file", "inbox-web"]); + assert.equal(r.files.copied.length, 3); assert.deepEqual( r.files.missing.map((m: any) => m.sourceId), ["gone-1"] @@ -199,7 +223,6 @@ test("migration imports, reports, and preserves data", async () => { assert.deepEqual(r.rendererKeys, [ "chat-web-1", "current-project", - "ingest-queue", "ks-table-rows", "theme", ]); @@ -232,13 +255,30 @@ test("migration imports, reports, and preserves data", async () => { assert.equal(asset.mediaType, "application/pdf"); assert.ok((await api.content(asset.id)).bytes.equals(fs.readFileSync(pdf))); + // Inbox entries keep their order, metadata, and managed file + const all = (await api.get("/v1/sources")).body.sources; + const inbox = all.filter((s: any) => s.projectId === null); + assert.deepEqual( + inbox.map((s: any) => s.id), + ["inbox-file", "inbox-web"] + ); + assert.deepEqual(inbox[1].data.topics, ["later"]); + assert.equal(byId["web-1"].title, "Fixture Site"); + const pendingAsset = (await api.get(`/v1/assets/${inbox[0].assetId}`)).body + .asset; + assert.equal(pendingAsset.filename, "pending.txt"); + assert.equal( + (await api.content(pendingAsset.id)).bytes.toString(), + "pending inbox file" + ); + const projects = (await api.get("/v1/projects")).body.projects; assert.equal(projects[0].data.icon, "pi pi-folder"); assert.equal(projects[0].data.description, "from renderer"); await svc.stop(); }); -test("a repeated migration creates no duplicates and keeps assets of deleted originals", () => { +test("a repeated migration creates no duplicates and keeps assets of deleted originals", async () => { const work = tempDir("rep-work"); const dataDir = tempDir("rep-data"); const { file, pdf, txt } = legacyBackup(work); @@ -253,6 +293,13 @@ test("a repeated migration creates no duplicates and keeps assets of deleted ori assert.equal(again.status, "completed"); assert.equal(again.projects.unchanged.length, 1); assert.equal(again.sources.unchanged.length, 5); + assert.deepEqual(again.inbox, []); + assert.deepEqual(again.sources.skipped.map((i: any) => i.id).sort(), [ + "inbox-file", + "inbox-web", + "orphan-1", + "web-1", + ]); assert.equal(again.sources.imported.length + again.sources.updated.length, 0); assert.equal(again.files.reused.length, 2); assert.deepEqual(counts(), before); @@ -261,6 +308,19 @@ test("a repeated migration creates no duplicates and keeps assets of deleted ori // Originals deleted: the sources keep their managed copies fs.rmSync(pdf); fs.rmSync(txt); + // An inbox entry moved to a project stays there on a repeated run + const svc = await startService(dataDir); + const api = client(svc); + const moved = (await api.get("/v1/sources")).body.sources.find( + (s: any) => s.id === "inbox-web" + ); + assert.equal( + (await api.put("/v1/sources/inbox-web", { ...moved, projectId: P1 })) + .status, + 200 + ); + await svc.stop(); + const third = report(migrate(dataDir, file).summary); assert.equal(third.files.kept.length, 2); assert.deepEqual( @@ -269,6 +329,14 @@ test("a repeated migration creates no duplicates and keeps assets of deleted ori ); assert.equal(third.sources.unchanged.length, 5); assert.deepEqual(counts(), before); + const db = new DatabaseSync(path.join(dataDir, "library.sqlite"), { + readOnly: true, + }); + const row = db + .prepare("SELECT project_id FROM sources WHERE id = 'inbox-web'") + .get() as any; + db.close(); + assert.equal(row.project_id, P1); }); test("a failed migration changes nothing and a later run succeeds", () => { @@ -291,7 +359,7 @@ test("a failed migration changes nothing and a later run succeeds", () => { const ok = migrate(dataDir, file); assert.equal(ok.status, 0); - assert.equal(count(dataDir, "sources"), 5); + assert.equal(count(dataDir, "sources"), 7); }); test("invalid input is rejected before any write", () => { @@ -336,7 +404,7 @@ test("dry run reports without writing", () => { assert.equal(result.status, 0); const summary = JSON.parse(result.stdout); assert.equal(summary.status, "dry-run"); - assert.equal(summary.sources.imported, 5); + assert.equal(summary.sources.imported, 7); assert.equal(count(dataDir, "sources"), 0); assert.ok(!fs.existsSync(path.join(dataDir, "migrations"))); }); @@ -347,8 +415,8 @@ test("the documented rollback restores the pre-migration state", async () => { const { file } = legacyBackup(work); const r = report(migrate(dataDir, file).summary); - assert.equal(count(dataDir, "sources"), 5); - assert.equal(assetFiles(dataDir).length, 2); + assert.equal(count(dataDir, "sources"), 7); + assert.equal(assetFiles(dataDir).length, 3); // Rollback: with the service stopped, replace the database with the snapshot for (const suffix of ["", "-wal", "-shm"]) { @@ -361,6 +429,6 @@ test("the documented rollback restores the pre-migration state", async () => { const api = client(svc); assert.deepEqual((await api.get("/v1/projects")).body.projects, []); assert.deepEqual(assetFiles(dataDir), []); - assert.match(svc.stderr(), /removed 2 unreferenced or temporary files/); + assert.match(svc.stderr(), /removed 3 unreferenced or temporary files/); await svc.stop(); }); diff --git a/src/kc_storage/test/restore.test.ts b/src/kc_storage/test/restore.test.ts index a61de0a8..a6e05581 100644 --- a/src/kc_storage/test/restore.test.ts +++ b/src/kc_storage/test/restore.test.ts @@ -119,15 +119,15 @@ test("restore through the API: validate, preview, activate, refuse a second time // An initialized but empty library is accepted assert.deepEqual((await api.get("/v1/library")).body, { empty: true, - counts: { projects: 0, sources: 0, assets: 0 }, + counts: { projects: 0, sources: 0, inbox: 0, assets: 0 }, }); const up = await upload(svc, tar); assert.equal(up.status, 201, JSON.stringify(up.body)); const preview = up.body.restore; - assert.deepEqual(preview.counts, { projects: 2, sources: 3, assets: 2, bytes: assets.pdf.size + assets.txt.size }); + assert.deepEqual(preview.counts, { projects: 2, sources: 3, inbox: 0, assets: 2, bytes: assets.pdf.size + assets.txt.size }); assert.deepEqual(preview.projectNames.sort(), ["Child", "Restore Project"]); - assert.equal(preview.backupVersion, 1); + assert.equal(preview.backupVersion, 2); assert.ok(preview.notIncluded.includes("Chat history")); assert.ok(preview.notIncluded.includes("API keys")); @@ -136,7 +136,7 @@ test("restore through the API: validate, preview, activate, refuse a second time const done = await activate(svc, preview.id); assert.equal(done.status, 200); - assert.deepEqual(done.body.restored, { projects: 2, sources: 3, assets: 2 }); + assert.deepEqual(done.body.restored, { projects: 2, sources: 3, inbox: 0, assets: 2 }); assert.deepEqual( (await api.get("/v1/projects")).body.projects.map(({ updatedAt, ...r }: any) => r), expected.projects @@ -196,8 +196,8 @@ test("invalid backups are rejected before activation and leave nothing behind", const cases: [string, Promise, RegExp][] = [ [ "unsupported version", - modified(tar, out("v2.tar"), (m) => (m.version = 2)), - /Backup version 2 is not supported/, + modified(tar, out("v3.tar"), (m) => (m.version = 3)), + /Backup version 3 is not supported/, ], [ "missing asset file", @@ -254,9 +254,9 @@ test("invalid backups are rejected before activation and leave nothing behind", assert.ok(!fs.existsSync("/tmp/kc-escaped.txt")); // The command line uses the same validation - const viaCli = cli(["restore", "--data-dir", tempDir("rs-cli"), "--from", out("v2.tar")]); + const viaCli = cli(["restore", "--data-dir", tempDir("rs-cli"), "--from", out("v3.tar")]); assert.equal(viaCli.status, 1); - assert.match(viaCli.stderr, /Backup version 2 is not supported/); + assert.match(viaCli.stderr, /Backup version 3 is not supported/); }); test("a failure during activation rolls back records and files", async () => { From e42a23e6987b432d22832d129dbdea0b2d395050 Mon Sep 17 00:00:00 2001 From: Rob Royce Date: Thu, 8 Oct 2026 09:44:08 -0700 Subject: [PATCH 2/5] feat(ui): service inbox, browser settings, session state, and logout Inbox (review finding 2): - The inbox loads from the storage service and saves each change there. An entry leaves the inbox list only after the transfer to a project is saved. A transfer that the service rejects stays visible on screen only where the service has it. - An inbox that an earlier version kept in localStorage moves to the service once with create-only writes. The key is removed only after every entry is saved. - Browser settings are a preference document in the service. The app loads them before startup, so no page saves defaults over them. - The launcher derives a stable port from the library path, so the browser address (and its UI preferences) stays the same. --port and --session-idle-minutes are new options. Session (review finding 3): - BackendService tracks the connection state (active, none, expired, signedOut, unreachable) from the session lifetime header and from 401, 403, and network failures. - StorageService writes run in one ordered queue of idempotent PUT and DELETE requests. A write refused for the session stays queued with all later writes and runs once after reconnect. Uploads repeat only after 401 or 403. The page warns before closing with queued writes. - A blocking session dialog explains recovery: paste a new launch link, or open it in another tab and check again. No reload is needed. - Log out in the title bar saves queued writes first (or asks), ends the session, clears library caches, and reloads without library data. - "Saved" in source details shows only after the service saved. Tests: Chrome tests for the inbox across a port change, an interrupted transfer, expiry and recovery, and logout, with console and page errors checked at each stage; a desktop test for the local inbox migration. Co-Authored-By: Claude Opus 5.5 --- e2e/browser/helpers.mjs | 183 +++++++++++ e2e/browser/inbox.e2e.mjs | 300 ++++++++++++++++++ e2e/browser/library.e2e.mjs | 173 ++++------ e2e/browser/session.e2e.mjs | 203 ++++++++++++ e2e/inbox.e2e.mjs | 125 ++++++++ e2e/steps.mjs | 12 +- scripts/start-browser.mjs | 34 +- src/kc_angular/src/app/app.component.html | 14 +- src/kc_angular/src/app/app.component.ts | 2 + src/kc_angular/src/app/app.module.ts | 11 +- .../message-templates/categorize.message.ts | 18 +- .../src/app/components/home.component.ts | 36 +-- .../settings/storage-settings.component.ts | 25 +- .../components/shared/session.component.ts | 256 +++++++++++++++ .../source-components/ks-details.component.ts | 6 +- .../src/app/platform/settings-store.ts | 129 +++++++- .../command-services/ks-command.service.ts | 8 +- .../factory-services/project.service.ts | 23 +- .../ingest-services/drag-and-drop.service.ts | 7 +- .../ingest-services/ingest.service.ts | 82 +++-- .../services/ipc-services/backend.service.ts | 289 +++++++++++++++-- .../services/ipc-services/storage.service.ts | 268 ++++++++++++++-- .../services/user-services/data.service.ts | 12 +- 23 files changed, 1932 insertions(+), 284 deletions(-) create mode 100644 e2e/browser/helpers.mjs create mode 100644 e2e/browser/inbox.e2e.mjs create mode 100644 e2e/browser/session.e2e.mjs create mode 100644 e2e/inbox.e2e.mjs create mode 100644 src/kc_angular/src/app/components/shared/session.component.ts diff --git a/e2e/browser/helpers.mjs b/e2e/browser/helpers.mjs new file mode 100644 index 00000000..1773e46c --- /dev/null +++ b/e2e/browser/helpers.mjs @@ -0,0 +1,183 @@ +/* + * Shared helpers for the Chrome tests: the "yarn browser" launcher, a + * Chrome page that records errors, and library reads through the page's + * own session. + */ + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { chromium } from "playwright-core"; +import { REPO } from "../lib.mjs"; + +const LINK = /http:\/\/127\.0\.0\.1:\d+\/#launch=[A-Za-z0-9_-]+/g; + +/** + * Start the launcher. Resolves with its first launch link. newLink() + * presses Enter in the launcher, as a user does, and resolves with the + * next link. + */ +export async function startLauncher(dataDir, log, args = []) { + const child = spawn( + process.execPath, + [ + path.join(REPO, "scripts/start-browser.mjs"), + "--data-dir", + dataDir, + ...args, + ], + { cwd: REPO, stdio: ["pipe", "pipe", "pipe"] } + ); + let output = ""; + const write = (d) => { + output += d; + fs.appendFileSync(log, d); + }; + child.stdout.on("data", write); + child.stderr.on("data", write); + + const nextLink = (count) => + new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`No launch link:\n${output}`)), + 30000 + ); + const check = () => { + const links = output.match(LINK) ?? []; + if (links.length >= count) { + clearTimeout(timer); + child.stdout.off("data", check); + resolve(links[count - 1]); + } + }; + child.stdout.on("data", check); + child.once("exit", (code) => + reject(new Error(`Launcher exited (${code}):\n${output}`)) + ); + check(); + }); + + const url = await nextLink(1); + return { + url, + origin: new URL(url).origin, + output: () => output, + async newLink() { + const count = (output.match(LINK) ?? []).length + 1; + child.stdin.write("\n"); + return nextLink(count); + }, + async stop() { + if (child.exitCode !== null) return; + child.kill("SIGINT"); + await new Promise((r) => child.on("exit", r)); + }, + }; +} + +/** + * Open Chrome on a launch link. errors collects uncaught page errors, + * console errors (with the URL that caused them), and alert dialogs. + */ +export async function openBrowser(url, { context: shared } = {}) { + const browser = shared + ? undefined + : await chromium.launch({ channel: "chrome", headless: true }); + const context = + shared ?? + (await browser.newContext({ + viewport: { width: 1600, height: 1000 }, + acceptDownloads: true, + })); + const page = await context.newPage(); + const errors = []; + page.on("pageerror", (e) => errors.push({ kind: "page", text: e.message })); + page.on("console", (m) => { + if (m.type() === "error") { + errors.push({ + kind: "console", + text: m.text(), + url: m.location()?.url ?? "", + }); + } + }); + page.on("dialog", (d) => { + errors.push({ kind: "dialog", text: d.message() }); + d.dismiss(); + }); + await page.goto(url); + await page.waitForSelector("app-create button", { timeout: 30000 }); + return { + browser, + context, + page, + errors, + async close() { + if (browser) await browser.close(); + else await page.close(); + }, + }; +} + +/** + * Fail on any recorded error that no pattern allows, then clear the list. + * A pattern matches the error text or the URL that caused it. + */ +export function assertNoErrors(errors, where, allowed = []) { + const unexpected = errors.filter( + (e) => !allowed.some((re) => re.test(e.text) || re.test(e.url ?? "")) + ); + errors.length = 0; + assert.deepEqual(unexpected, [], `errors ${where}`); +} + +/** Projects, sources, and the inbox through the page's own session. */ +export function readLibrary(page) { + return page.evaluate(async () => { + const get = (r) => fetch(`/v1/${r}`).then((res) => res.json()); + const { projects } = await get("projects"); + const { sources } = await get("sources"); + const view = (s) => ({ + title: s.title, + type: s.ingestType, + assetId: s.assetId, + originalPath: s.data.reference?.source?.file?.path ?? null, + topics: s.data.topics ?? [], + annotations: (s.data.meta ?? []) + .filter((m) => m.key === "annotation") + .map((m) => m.value), + }); + return { + projects: projects.map((p) => ({ + name: p.name, + sources: sources + .filter((s) => s.projectId === p.id) + .map(view) + .sort((a, b) => a.title.localeCompare(b.title)), + })), + inbox: sources.filter((s) => s.projectId === null).map(view), + }; + }); +} + +/** Read the library with the bearer-free page session, as JSON counts. */ +export async function counts(page) { + return page.evaluate(() => + fetch("/v1/library") + .then((r) => r.json()) + .then((b) => b.counts) + ); +} + +/** Poll a value that the page reads (it may await), until check passes. */ +export async function waitForValue(page, read, check, timeout = 15000) { + const end = Date.now() + timeout; + let value; + while (Date.now() < end) { + value = await page.evaluate(read); + if (check(value)) return value; + await new Promise((r) => setTimeout(r, 300)); + } + throw new Error(`Timed out. Last value: ${JSON.stringify(value)}`); +} diff --git a/e2e/browser/inbox.e2e.mjs b/e2e/browser/inbox.e2e.mjs new file mode 100644 index 00000000..79d2187d --- /dev/null +++ b/e2e/browser/inbox.e2e.mjs @@ -0,0 +1,300 @@ +/* + * Inbox, preferences, and platform limits in Google Chrome, without + * Electron. Each test uses its own scratch library. + * + * Build first: yarn build-angular-dev + * Run: yarn e2e-browser + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { newRun } from "../lib.mjs"; +import * as ui from "../steps.mjs"; +import { + assertNoErrors, + counts, + openBrowser, + readLibrary, + startLauncher, + waitForValue, +} from "./helpers.mjs"; + +/** A fresh library has no saved browser settings yet. */ +const FIRST_START = [/\/v1\/preferences\/browser-settings$/]; + +/** Requests that the service refuses while the session is broken. */ +const REFUSED = [/status of 401/]; + +function scratchFiles(run, names) { + return names.map((name) => { + const file = path.join(run, name); + fs.writeFileSync(file, `content of ${name}`); + return file; + }); +} + +async function uploadToInbox(page, files) { + await page.locator("app-create input[type=file]").setInputFiles(files); + await ui.openInbox(page); + for (const file of files) { + await page + .getByText(path.basename(file)) + .first() + .waitFor({ timeout: 15000 }); + } +} + +/** Replace the session cookie, so the service refuses the next request. */ +async function breakSession(context, origin) { + const port = new URL(origin).port; + await context.addCookies([ + { name: `kc_session_${port}`, value: "x".repeat(43), url: origin }, + ]); +} + +/** Paste a new launch link into the session dialog. */ +async function continueWithLink(page, link) { + const dialog = page.locator(".session-dialog"); + await dialog.locator("[data-test=session-link]").fill(link); + await dialog.locator("[data-test=session-continue]").click(); + await dialog.waitFor({ state: "hidden", timeout: 15000 }); +} + +async function autoplaySwitch(page) { + await ui.openSettings(page, "Display"); + const control = page + .locator("app-setting-template", { hasText: "Auto-play YouTube videos" }) + .locator(".p-inputswitch"); + await control.waitFor({ timeout: 15000 }); + return control; +} + +test( + "inbox entries and preferences survive a restart on a different port", + { timeout: 300000 }, + async () => { + const run = newRun("browser-inbox"); + const library = path.join(run, "library"); + const log = path.join(run, "launcher.log"); + let launcher; + let b; + try { + launcher = await startLauncher(library, log); + b = await openBrowser(launcher.url); + const files = scratchFiles(run, ["first.txt", "second.txt"]); + await uploadToInbox(b.page, files); + + // Change a preference in Settings > Display + const toggle = await autoplaySwitch(b.page); + const before = (await toggle.getAttribute("class")).includes( + "p-inputswitch-checked" + ); + await toggle.click(); + await waitForValue( + b.page, + () => + fetch("/v1/preferences/browser-settings") + .then((r) => r.json()) + .then((j) => j.preference?.data.display?.autoplay), + (value) => value === !before + ); + const saved = await readLibrary(b.page); + assert.deepEqual( + saved.inbox.map((s) => s.title), + ["first.txt", "second.txt"] + ); + assertNoErrors(b.errors, "before the restart", FIRST_START); + await b.close(); + b = undefined; + await launcher.stop(); + + // Same library, different port: a new browser address + const oldPort = Number(new URL(launcher.url).port); + launcher = await startLauncher(library, log, [ + "--port", + String(oldPort + 1), + ]); + assert.notEqual(new URL(launcher.url).port, String(oldPort)); + b = await openBrowser(launcher.url); + assert.deepEqual(await readLibrary(b.page), saved); + await ui.openInbox(b.page); + for (const name of ["first.txt", "second.txt"]) { + await b.page.getByText(name).first().waitFor({ timeout: 15000 }); + } + const after = await autoplaySwitch(b.page); + assert.equal( + (await after.getAttribute("class")).includes("p-inputswitch-checked"), + !before, + "the preference survives the address change" + ); + assertNoErrors(b.errors, "after the restart"); + } finally { + if (b) await b.close(); + if (launcher) await launcher.stop(); + } + } +); + +test( + "an inbox transfer neither loses nor duplicates a source", + { timeout: 300000 }, + async () => { + const run = newRun("browser-transfer"); + let launcher; + let b; + try { + launcher = await startLauncher( + path.join(run, "library"), + path.join(run, "launcher.log") + ); + b = await openBrowser(launcher.url); + await ui.createProject(b.page, "Target Project"); + const [file] = scratchFiles(run, ["move-me.txt"]); + await uploadToInbox(b.page, [file]); + const entry = (await readLibrary(b.page)).inbox[0]; + assertNoErrors(b.errors, "before the transfer", FIRST_START); + + // The service refuses the transfer: nothing moves, nothing is lost + await breakSession(b.context, launcher.origin); + await b.page.getByText("Import All").click(); + await b.page.locator("button", { hasText: "Import" }).first().click(); + const dialog = b.page.locator(".session-dialog"); + await dialog.waitFor({ timeout: 15000 }); + await dialog + .locator("[data-test=session-unsaved]") + .waitFor({ timeout: 15000 }); + assert.ok( + await b.page.locator("app-home").getByText("move-me.txt").count(), + "the entry stays in the inbox until the transfer is saved" + ); + + const observer = await openBrowser(await launcher.newLink()); + const pending = await readLibrary(observer.page); + assert.deepEqual( + pending.inbox.map((s) => s.title), + ["move-me.txt"] + ); + assert.deepEqual(pending.projects[0].sources, []); + await observer.close(); + + // A new session saves the held transfer once + await continueWithLink(b.page, await launcher.newLink()); + await b.page + .locator("[data-test=session-saving]") + .waitFor({ state: "detached", timeout: 15000 }); + const moved = await readLibrary(b.page); + assert.deepEqual(moved.inbox, []); + assert.deepEqual( + moved.projects[0].sources.map((s) => [s.title, s.assetId]), + [["move-me.txt", entry.assetId]] + ); + + // Repeating the transfer request adds nothing + const repeat = await b.page.evaluate(async () => { + const csrf = await fetch("/v1/session") + .then((r) => r.json()) + .then((s) => s.csrfToken); + const { sources } = await fetch("/v1/sources").then((r) => r.json()); + const source = sources[0]; + const res = await fetch(`/v1/sources/${source.id}`, { + method: "PUT", + headers: { + "Content-Type": "application/json", + "X-Knowledge-CSRF": csrf, + }, + body: JSON.stringify(source), + }); + return res.status; + }); + assert.equal(repeat, 200); + assert.deepEqual(await counts(b.page), { + projects: 1, + sources: 1, + inbox: 0, + assets: 1, + }); + await ui.selectProject(b.page, "Target Project"); + await ui.openTable(b.page); + assert.deepEqual(await ui.tableTitles(b.page), ["move-me.txt"]); + assertNoErrors(b.errors, "after the transfer", REFUSED); + } finally { + if (b) await b.close(); + if (launcher) await launcher.stop(); + } + } +); + +test( + "Save as PDF is unavailable in Chrome and says why", + { timeout: 300000 }, + async () => { + const run = newRun("browser-pdf"); + let launcher; + let b; + try { + launcher = await startLauncher( + path.join(run, "library"), + path.join(run, "launcher.log") + ); + b = await openBrowser(launcher.url); + await ui.createProject(b.page, "Web Project"); + + // A website source, as the desktop app saves it + await b.page.evaluate(async () => { + const csrf = await fetch("/v1/session") + .then((r) => r.json()) + .then((s) => s.csrfToken); + const { projects } = await fetch("/v1/projects").then((r) => r.json()); + const link = "https://example.com/"; + await fetch("/v1/sources/web-1", { + method: "PUT", + headers: { + "Content-Type": "application/json", + "X-Knowledge-CSRF": csrf, + }, + body: JSON.stringify({ + projectId: projects[0].id, + title: "Example Website", + ingestType: "website", + assetId: null, + data: { + accessLink: link, + reference: { + ingestType: "website", + source: { website: { accessLink: link } }, + link, + }, + topics: [], + meta: [], + dateCreated: new Date().toISOString(), + dateAccessed: [], + dateModified: [], + }, + }), + }); + }); + await b.page.reload(); + await b.page.waitForSelector("app-create button"); + await ui.selectProject(b.page, "Web Project"); + await b.page.locator("i.pi-th-large").first().click(); + const card = b.page.locator("app-ks-card", { + hasText: "Example Website", + }); + await card.hover(); + + const button = card.locator("[data-test=save-pdf]"); + await button.waitFor({ timeout: 15000 }); + assert.ok(await button.isDisabled()); + assert.match(await button.getAttribute("aria-label"), /desktop app/); + await button.click({ force: true }); + await b.page.waitForTimeout(1500); + assert.equal(await b.page.getByText("Creating PDF").count(), 0); + assertNoErrors(b.errors, "after Save as PDF", FIRST_START); + } finally { + if (b) await b.close(); + if (launcher) await launcher.stop(); + } + } +); diff --git a/e2e/browser/library.e2e.mjs b/e2e/browser/library.e2e.mjs index 137fb471..11b30ac2 100644 --- a/e2e/browser/library.e2e.mjs +++ b/e2e/browser/library.e2e.mjs @@ -5,8 +5,9 @@ * with a scratch library, opens the printed launch link in Chrome, and runs: * create a project, upload a PDF and a text file, show both, annotate, * search, restart the service and reopen the browser, open the managed - * files, export the library, and restore it into a second, empty library. - * It also checks unauthorized and cross-origin requests. + * files, keep a file in the inbox, export the library, and restore it + * into a second, empty library. It also checks unauthorized and + * cross-origin requests, and console and page errors at each stage. * * Build first: yarn build-angular-dev * Run: yarn e2e-browser @@ -16,104 +17,23 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; -import { spawn } from "node:child_process"; -import { chromium } from "playwright-core"; -import { FIXTURES, REPO, newRun, startFixtureSite } from "../lib.mjs"; +import { FIXTURES, newRun, startFixtureSite } from "../lib.mjs"; import * as ui from "../steps.mjs"; +import { + assertNoErrors, + counts, + openBrowser, + readLibrary, + startLauncher, +} from "./helpers.mjs"; const PROJECT = "Browser Project"; const PDF = "recovery-fixture.pdf"; const TXT = "recovery-note.txt"; +const PENDING = "pending-inbox.txt"; -/** Start the launcher. Resolves with its first launch link and output. */ -async function startLauncher(dataDir, log) { - const child = spawn( - process.execPath, - [path.join(REPO, "scripts/start-browser.mjs"), "--data-dir", dataDir], - { - cwd: REPO, - stdio: ["pipe", "pipe", "pipe"], - } - ); - let output = ""; - const write = (d) => { - output += d; - fs.appendFileSync(log, d); - }; - child.stdout.on("data", write); - child.stderr.on("data", write); - const url = await new Promise((resolve, reject) => { - const timer = setTimeout( - () => reject(new Error(`No launch link:\n${output}`)), - 30000 - ); - child.stdout.on("data", () => { - const m = output.match( - /http:\/\/127\.0\.0\.1:\d+\/#launch=[A-Za-z0-9_-]+/ - ); - if (m) { - clearTimeout(timer); - resolve(m[0]); - } - }); - child.on("exit", (code) => - reject(new Error(`Launcher exited (${code}):\n${output}`)) - ); - }); - return { - url, - origin: new URL(url).origin, - output: () => output, - async stop() { - if (child.exitCode !== null) return; - child.kill("SIGINT"); - await new Promise((r) => child.on("exit", r)); - }, - }; -} - -/** Projects and sources through the page's own session (same origin). */ -function readLibrary(page) { - return page.evaluate(async () => { - const get = (r) => fetch(`/v1/${r}`).then((res) => res.json()); - const { projects } = await get("projects"); - const { sources } = await get("sources"); - return projects.map((p) => ({ - name: p.name, - sources: sources - .filter((s) => s.projectId === p.id) - .map((s) => ({ - title: s.title, - type: s.ingestType, - assetId: s.assetId, - originalPath: s.data.reference?.source?.file?.path ?? null, - topics: s.data.topics ?? [], - annotations: (s.data.meta ?? []) - .filter((m) => m.key === "annotation") - .map((m) => m.value), - })) - .sort((a, b) => a.title.localeCompare(b.title)), - })); - }); -} - -async function openBrowser(url) { - const browser = await chromium.launch({ channel: "chrome", headless: true }); - const context = await browser.newContext({ - viewport: { width: 1600, height: 1000 }, - acceptDownloads: true, - }); - const page = await context.newPage(); - const errors = []; - page.on("pageerror", (e) => errors.push(e.message)); - page.on("dialog", (d) => { - errors.push(`dialog: ${d.message()}`); - d.dismiss(); - }); - await page.goto(url); - await page.waitForSelector("app-create button", { timeout: 30000 }); - return { browser, context, page, errors }; -} +/** A fresh library has no saved browser settings yet. */ +const FIRST_START = [/\/v1\/preferences\/browser-settings$/]; test( "library workflow in Chrome without Electron", @@ -123,8 +43,6 @@ test( const libraryA = path.join(run, "library-a"); const libraryB = path.join(run, "library-b"); const log = path.join(run, "launcher.log"); - const shot = (page, name) => - page.screenshot({ path: path.join(run, `${name}.png`) }); const pdfBytes = fs.readFileSync(path.join(FIXTURES, PDF)); const txtBytes = fs.readFileSync(path.join(FIXTURES, TXT)); const other = await startFixtureSite(); @@ -170,7 +88,6 @@ test( ) ); await b.page.waitForTimeout(1500); - await shot(b.page, `a-document-${name}`); await ui.closeDialog(b.page); } @@ -189,11 +106,12 @@ test( ); const library = await readLibrary(b.page); - assert.equal(library.length, 1); - const note = library[0].sources.find((s) => s.title === TXT); + assert.equal(library.projects.length, 1); + assert.deepEqual(library.inbox, []); + const note = library.projects[0].sources.find((s) => s.title === TXT); assert.deepEqual(note.topics, ["browsertopic"]); assert.deepEqual(note.annotations, ["browser note"]); - for (const s of library[0].sources) { + for (const s of library.projects[0].sources) { assert.match(s.assetId, /^[0-9a-f-]{36}$/); assert.equal(s.originalPath, "", "the browser sends no local path"); } @@ -250,18 +168,16 @@ test( await attacker.close(); assert.equal(attack.read, "blocked"); assert.equal(attack.write, "blocked"); - const status = await b.page.evaluate(() => - fetch("/v1/library").then((r) => r.json()) - ); assert.deepEqual( - status.counts, - { projects: 1, sources: 2, assets: 2 }, + await counts(b.page), + { projects: 1, sources: 2, inbox: 0, assets: 2 }, "no planted records or files" ); // The launcher never prints the bearer token assert.doesNotMatch(launcher.output(), /[0-9a-f]{64}/); - await b.browser.close(); + assertNoErrors(b.errors, "before the restart", FIRST_START); + await b.close(); b = undefined; // 7. Restart the service and reopen the browser @@ -279,7 +195,7 @@ test( [PDF, pdfBytes], [TXT, txtBytes], ]) { - const assetId = library[0].sources.find( + const assetId = library.projects[0].sources.find( (s) => s.title === name ).assetId; const popup = b.context.waitForEvent("page"); @@ -300,13 +216,30 @@ test( await tab.close(); } + // A file stays in the inbox. The backup must keep it. + const pending = path.join(run, PENDING); + fs.writeFileSync(pending, "pending inbox file"); + await b.page + .locator("app-create input[type=file]") + .setInputFiles(pending); + await ui.openInbox(b.page); + await b.page.getByText(PENDING).first().waitFor({ timeout: 15000 }); + await b.page.waitForTimeout(1500); + const withInbox = await readLibrary(b.page); + assert.deepEqual( + withInbox.inbox.map((s) => s.title), + [PENDING] + ); + assertNoErrors(b.errors, "after the restart"); + // 10. Export the library through the UI await ui.openSettings(b.page, "Backup"); const download = b.page.waitForEvent("download"); await b.page.locator("button", { hasText: "Export Library" }).click(); const tar = path.join(run, "library.tar"); await (await download).saveAs(tar); - await b.browser.close(); + assertNoErrors(b.errors, "during export"); + await b.close(); b = undefined; await launcher.stop(); @@ -314,23 +247,35 @@ test( launcher = await startLauncher(libraryB, log); b = await openBrowser(launcher.url); const restore = await ui.restoreLibrary(b.page, tar); - assert.match(restore.preview, /1 project, 2 sources, 2 files/); + assert.match( + restore.preview, + /1 project, 3 sources \(1 inbox entry\), 3 files/ + ); assert.match( restore.result, - /Restored 1 project, 2 sources, and 2 files/ + /Restored 1 project, 3 sources \(1 inbox entry\), and 3 files/ ); - assert.deepEqual(await readLibrary(b.page), library); + assert.deepEqual(await readLibrary(b.page), withInbox); + await ui.selectProject(b.page, PROJECT); await ui.openTable(b.page); await ui.openSource(b.page, PDF); await ui.openDocumentTab(b.page); await b.page.locator("source-document embed").waitFor({ timeout: 15000 }); await b.page.waitForTimeout(1500); - await shot(b.page, "b-restored-pdf"); + await ui.closeDialog(b.page); + + // The restored inbox shows the entry, and its file has the same bytes + await ui.openInbox(b.page); + await b.page.getByText(PENDING).first().waitFor({ timeout: 15000 }); + const restoredFile = await b.context.request.get( + `${launcher.origin}/v1/assets/${withInbox.inbox[0].assetId}/content/${PENDING}` + ); + assert.equal(await restoredFile.text(), "pending inbox file"); - assert.deepEqual(b.errors, [], "page errors or dialogs"); + assertNoErrors(b.errors, "after the restore", FIRST_START); } finally { - if (b) await b.browser.close(); + if (b) await b.close(); if (launcher) await launcher.stop(); other.close(); } diff --git a/e2e/browser/session.e2e.mjs b/e2e/browser/session.e2e.mjs new file mode 100644 index 00000000..b7e1d21c --- /dev/null +++ b/e2e/browser/session.e2e.mjs @@ -0,0 +1,203 @@ +/* + * Browser session expiry and logout in Google Chrome, without Electron. + * + * Build first: yarn build-angular-dev + * Run: yarn e2e-browser + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { newRun } from "../lib.mjs"; +import * as ui from "../steps.mjs"; +import { + assertNoErrors, + openBrowser, + readLibrary, + startLauncher, +} from "./helpers.mjs"; + +/** A fresh library has no saved browser settings yet. */ +const FIRST_START = [/\/v1\/preferences\/browser-settings$/]; + +/** Projects and inbox titles, read with a separate session. */ +async function serverView(launcher) { + const observer = await openBrowser(await launcher.newLink()); + try { + const library = await readLibrary(observer.page); + return { + projects: library.projects.map((p) => p.name).sort(), + inbox: library.inbox.map((s) => s.title), + }; + } finally { + await observer.close(); + } +} + +test( + "session expiry blocks changes and keeps unsaved work; a new session saves it once", + { timeout: 300000 }, + async () => { + const run = newRun("browser-expiry"); + let launcher; + let b; + try { + // Sessions end after 12 seconds without requests + launcher = await startLauncher( + path.join(run, "library"), + path.join(run, "launcher.log"), + ["--session-idle-minutes", "0.2"] + ); + b = await openBrowser(launcher.url); + await ui.createProject(b.page, "First Project"); + + // Start an edit, then let the session end + await b.page.locator("button.create-project").click(); + const form = b.page + .locator(".p-dialog") + .filter({ hasText: "Create Project" }); + await form.locator("input").first().fill("Pending Project"); + const dialog = b.page.locator(".session-dialog"); + await dialog.waitFor({ timeout: 30000 }); + assert.match(await dialog.innerText(), /Session ended/); + assert.match(await dialog.innerText(), /press Enter for a new link/); + + // No change is possible while the session is invalid + await assert.rejects( + form.locator("button", { hasText: "Create" }).click({ timeout: 1500 }), + /intercepts pointer events|Timeout/ + ); + assert.equal( + await form.locator("input").first().inputValue(), + "Pending Project", + "the unsaved edit stays" + ); + assert.deepEqual((await serverView(launcher)).projects, [ + "First Project", + ]); + + // Recover without a reload: open a new link in another tab, then + // select Check again + const other = await openBrowser(await launcher.newLink(), { + context: b.context, + }); + await other.close(); + await dialog.locator("[data-test=session-check]").click(); + await dialog.waitFor({ state: "hidden", timeout: 15000 }); + + // The pending edit completes once + await form.locator("button", { hasText: "Create" }).click(); + await b.page + .locator("p-tree, .p-tree") + .getByText("Pending Project", { exact: true }) + .first() + .waitFor({ timeout: 15000 }); + assert.deepEqual((await serverView(launcher)).projects, [ + "First Project", + "Pending Project", + ]); + assertNoErrors(b.errors, "around the expiry", FIRST_START); + } finally { + if (b) await b.close(); + if (launcher) await launcher.stop(); + } + } +); + +test( + "logout saves queued changes first, then ends access and clears the page", + { timeout: 300000 }, + async () => { + const run = newRun("browser-logout"); + let launcher; + let b; + try { + launcher = await startLauncher( + path.join(run, "library"), + path.join(run, "launcher.log") + ); + b = await openBrowser(launcher.url); + await ui.createProject(b.page, "Secret Project"); + const file = path.join(run, "secret-inbox.txt"); + fs.writeFileSync(file, "secret inbox file"); + await b.page.locator("app-create input[type=file]").setInputFiles(file); + await ui.openInbox(b.page); + await b.page.getByText("secret-inbox.txt").first().waitFor(); + + // A slow write is still running when the user logs out + await b.page.route("**/v1/projects/*", async (route) => { + if (route.request().method() === "PUT") { + await new Promise((r) => setTimeout(r, 2000)); + } + await route.continue(); + }); + await b.page.locator("button.create-project").click(); + const form = b.page + .locator(".p-dialog") + .filter({ hasText: "Create Project" }); + await form.locator("input").first().fill("Late Project"); + await form.locator("button", { hasText: "Create" }).click(); + await b.page.locator("[data-test=logout]").click(); + + // Logged out: the page shows no library data and has no access + const dialog = b.page.locator(".session-dialog"); + await dialog.waitFor({ timeout: 30000 }); + assert.match(await dialog.innerText(), /Logged out/); + for (const text of [ + "Secret Project", + "Late Project", + "secret-inbox.txt", + ]) { + assert.equal(await b.page.getByText(text).count(), 0, text); + } + assert.equal( + await b.page.evaluate(() => + fetch("/v1/projects").then((r) => r.status) + ), + 401 + ); + const port = new URL(launcher.origin).port; + const cookies = await b.context.cookies(launcher.origin); + assert.equal( + cookies.filter((c) => c.name === `kc_session_${port}` && c.value) + .length, + 0, + "the session cookie is gone" + ); + const cached = await b.page.evaluate(() => + Object.keys(localStorage).filter( + (k) => k.startsWith("icon-") || k === "current-project" + ) + ); + assert.deepEqual(cached, []); + + // The queued change was saved before the session ended + assert.deepEqual(await serverView(launcher), { + projects: ["Late Project", "Secret Project"], + inbox: ["secret-inbox.txt"], + }); + + // A new link opens the library again + await dialog + .locator("[data-test=session-link]") + .fill(await launcher.newLink()); + await Promise.all([ + b.page.waitForEvent("load"), + dialog.locator("[data-test=session-continue]").click(), + ]); + await b.page + .locator("p-tree, .p-tree") + .getByText("Secret Project", { exact: true }) + .first() + .waitFor({ timeout: 15000 }); + assertNoErrors(b.errors, "around the logout", [ + ...FIRST_START, + /status of 401/, + ]); + } finally { + if (b) await b.close(); + if (launcher) await launcher.stop(); + } + } +); diff --git a/e2e/inbox.e2e.mjs b/e2e/inbox.e2e.mjs new file mode 100644 index 00000000..45a7b7df --- /dev/null +++ b/e2e/inbox.e2e.mjs @@ -0,0 +1,125 @@ +/* + * Desktop inbox: an inbox that an earlier version kept in renderer + * localStorage moves to the storage service once, and inbox entries + * survive restarts. + * + * Run: yarn build-dev && yarn e2e + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { close, FIXTURES, launch, newRun, readAsset } from "./lib.mjs"; +import * as ui from "./steps.mjs"; + +/** Inbox entries (sources without a project) from this instance's service. */ +function readInbox(page) { + return page.evaluate(async () => { + const { storage } = await window.api.invoke("A2E:Backend:Info"); + const res = await fetch(`${storage.url}/v1/sources`, { + headers: { Authorization: `Bearer ${storage.token}` }, + }); + const { sources } = await res.json(); + return sources + .filter((s) => s.projectId === null) + .map((s) => ({ id: s.id, title: s.title, assetId: s.assetId })); + }); +} + +test( + "an earlier local inbox moves to the service once and survives restarts", + { timeout: 300000 }, + async () => { + const run = newRun("desktop-inbox"); + const profile = path.join(run, "profile"); + const log = path.join(run, "app.log"); + const legacyFile = path.join(run, "legacy-note.txt"); + fs.copyFileSync(path.join(FIXTURES, "recovery-note.txt"), legacyFile); + + // An earlier version left two entries in localStorage + let ctx = await launch(profile, log); + await ctx.page.evaluate((file) => { + localStorage.setItem( + "ingest-queue", + JSON.stringify([ + { + id: { value: "legacy-file" }, + title: "legacy-note.txt", + ingestType: "file", + accessLink: file, + associatedProject: { value: "" }, + reference: { + ingestType: "file", + source: { + file: { + filename: "legacy-note.txt", + path: file, + size: 1, + type: "text/plain", + }, + }, + link: file, + }, + dateCreated: "2026-10-01T00:00:00.000Z", + topics: [], + }, + { + id: { value: "legacy-web" }, + title: "Legacy Website", + ingestType: "website", + accessLink: "https://example.com/", + associatedProject: { value: "" }, + dateCreated: "2026-10-01T00:00:01.000Z", + topics: ["kept"], + }, + ]) + ); + }, legacyFile); + await close(ctx); + + // Next start: the entries move to the service; the old key is removed + ctx = await launch(profile, log); + await ctx.page.waitForFunction( + () => localStorage.getItem("ingest-queue") === null, + null, + { timeout: 15000 } + ); + const moved = await readInbox(ctx.page); + assert.deepEqual( + moved.map((s) => s.title), + ["legacy-note.txt", "Legacy Website"] + ); + assert.match(moved[0].assetId, /^[0-9a-f-]{36}$/, "the file is copied"); + assert.ok( + (await readAsset(ctx.page, moved[0].assetId)).equals( + fs.readFileSync(legacyFile) + ) + ); + + // A file added to the inbox now is saved by the service + await ctx.page + .locator("app-create input[type=file]") + .setInputFiles(path.join(FIXTURES, "recovery-fixture.pdf")); + await ui.openInbox(ctx.page); + await ctx.page.getByText("recovery-fixture.pdf").first().waitFor(); + await ctx.page.waitForTimeout(1500); + const withNew = await readInbox(ctx.page); + assert.equal(withNew.length, 3); + await close(ctx); + + // A repeated start changes nothing and shows every entry + fs.rmSync(legacyFile); + ctx = await launch(profile, log); + assert.deepEqual(await readInbox(ctx.page), withNew); + await ui.openInbox(ctx.page); + for (const title of [ + "legacy-note.txt", + "Legacy Website", + "recovery-fixture.pdf", + ]) { + await ctx.page.getByText(title).first().waitFor({ timeout: 15000 }); + } + await close(ctx); + } +); diff --git a/e2e/steps.mjs b/e2e/steps.mjs index 7e85f7a0..8da8c062 100644 --- a/e2e/steps.mjs +++ b/e2e/steps.mjs @@ -59,6 +59,15 @@ export async function importFilesToProject(page, files) { await page.locator("button", { hasText: "Import" }).first().click(); } +/** Show the Inbox (keyboard shortcut Cmd+1 / Ctrl+1). */ +export async function openInbox(page) { + await page.locator("body").click({ position: { x: 5, y: 600 } }); + await page.keyboard.press( + process.platform === "darwin" ? "Meta+1" : "Control+1" + ); + await page.locator("app-home").first().waitFor({ timeout: TIMEOUT }); +} + /** Select a project in the project tree. */ export async function selectProject(page, name) { await page @@ -125,8 +134,9 @@ export async function addTopic(page, topic) { await input.press("Enter"); } +/** The Document tab of the open source dialog. */ export async function openDocumentTab(page) { - await page.getByText("Document", { exact: true }).click(); + await page.getByRole("dialog").getByText("Document", { exact: true }).click(); } /** Type into the global search box and return the suggestion texts. */ diff --git a/scripts/start-browser.mjs b/scripts/start-browser.mjs index feee5c90..be7e1c1b 100644 --- a/scripts/start-browser.mjs +++ b/scripts/start-browser.mjs @@ -1,11 +1,16 @@ /* * Start the storage service with the browser UI. No Electron. * - * yarn browser [--data-dir ] [--web-root ] [--open] + * yarn browser [--data-dir ] [--web-root ] [--port ] + * [--session-idle-minutes ] [--open] * * Defaults: --data-dir .dev-profiles/browser/library, * --web-root src/kc_angular/dist/main (run yarn build-angular-dev first). * + * The port comes from the library path, so one library always opens at the + * same browser address. The page and its browser preferences belong to + * that address. Use --port when that port is in use. + * * The bearer token stays in this process and the service environment. It * is never printed. The script prints a one-time launch link. Press Enter * for a new link (for example, after a session ends). Press Ctrl+C to stop. @@ -28,10 +33,22 @@ const { values } = parseArgs({ type: "string", default: path.join(ROOT, "src", "kc_angular", "dist", "main"), }, + port: { type: "string" }, + "session-idle-minutes": { type: "string" }, open: { type: "boolean", default: false }, }, }); +const dataDir = path.resolve(values["data-dir"]); + +/** 41000-48999: below the ephemeral range of macOS, Linux, and Windows. */ +function libraryPort(dir) { + const hash = crypto.createHash("sha256").update(dir).digest(); + return 41000 + (hash.readUInt32BE(0) % 8000); +} + +const port = values.port ?? String(libraryPort(dataDir)); + const token = crypto.randomBytes(32).toString("hex"); const service = spawn( process.execPath, @@ -39,9 +56,14 @@ const service = spawn( path.join(ROOT, "src", "kc_storage", "src", "main.ts"), "serve", "--data-dir", - path.resolve(values["data-dir"]), + dataDir, "--web-root", path.resolve(values["web-root"]), + "--port", + port, + ...(values["session-idle-minutes"] + ? ["--session-idle-minutes", values["session-idle-minutes"]] + : []), "--exit-on-stdin-close", ], { @@ -58,7 +80,13 @@ const ready = await new Promise((resolve, reject) => { if (out.includes("\n")) resolve(JSON.parse(line)); }); service.on("exit", (code) => - reject(new Error(`The storage service stopped (code ${code}).`)) + reject( + new Error( + `The storage service stopped (code ${code}). If port ${port} is in use, ` + + "stop the other program or choose a port with --port . " + + "A different port is a different browser address, with its own browser preferences." + ) + ) ); }); diff --git a/src/kc_angular/src/app/app.component.html b/src/kc_angular/src/app/app.component.html index bf52f0d9..45b236ce 100755 --- a/src/kc_angular/src/app/app.component.html +++ b/src/kc_angular/src/app/app.component.html @@ -45,7 +45,13 @@ -
+ +
+ + - DESKTOP ? new DesktopSettingsStore() : new BrowserSettingsStore(), + useClass: DESKTOP ? DesktopSettingsStore : BrowserSettingsStore, }, { provide: WindowControls, diff --git a/src/kc_angular/src/app/components/chat-components/message-templates/categorize.message.ts b/src/kc_angular/src/app/components/chat-components/message-templates/categorize.message.ts index 85ac58d6..221e4b42 100644 --- a/src/kc_angular/src/app/components/chat-components/message-templates/categorize.message.ts +++ b/src/kc_angular/src/app/components/chat-components/message-templates/categorize.message.ts @@ -265,19 +265,11 @@ export class CategorizeMessage implements OnInit { ]); } else { // Otherwise it must be in the inbox - this.service - .updateProjects([ - { - id: { value: project.id }, - addKnowledgeSource: [source], - }, - ]) - .then(() => { - setTimeout(() => { - this.ingest.add(source); - this.service.setCurrentProject(project.id); - }, 500); - }); + this.ingest.transfer([source], project.id).then((saved) => { + if (saved) { + setTimeout(() => this.service.setCurrentProject(project.id), 500); + } + }); } }); } diff --git a/src/kc_angular/src/app/components/home.component.ts b/src/kc_angular/src/app/components/home.component.ts index cbdcbb8b..e95df22a 100755 --- a/src/kc_angular/src/app/components/home.component.ts +++ b/src/kc_angular/src/app/components/home.component.ts @@ -455,34 +455,14 @@ export class HomeComponent implements OnDestroy { return; } - if (this.importAll) { - this.projects - .updateProjects([ - { - id: project.id, - addKnowledgeSource: this.upNext, - }, - ]) - .then(() => { - if (this.active) { - for (const source of this.upNext) { - this.ingest.add(source); - } - } - }); - } else if (this.active) { - this.projects - .updateProjects([ - { - id: project.id, - addKnowledgeSource: [this.active], - }, - ]) - .then(() => { - if (this.active) { - this.ingest.add(this.active); - } - }); + // An entry leaves the inbox only after the service saved the move + const sources = this.importAll + ? [...this.upNext] + : this.active + ? [this.active] + : []; + if (sources.length > 0) { + this.ingest.transfer(sources, project.id.value); } } diff --git a/src/kc_angular/src/app/components/settings/storage-settings.component.ts b/src/kc_angular/src/app/components/settings/storage-settings.component.ts index 0b74b336..139f67c9 100755 --- a/src/kc_angular/src/app/components/settings/storage-settings.component.ts +++ b/src/kc_angular/src/app/components/settings/storage-settings.component.ts @@ -36,8 +36,8 @@ import { SettingsService } from '@services/ipc-services/settings.service';
- Contains projects, sources, topics, metadata, and copies of - imported files. Does not contain chat history, the inbox, UI + Contains projects, sources, the inbox, topics, metadata, and + copies of imported files. Does not contain chat history, UI preferences, settings, or API keys.
@@ -92,7 +92,8 @@ import { SettingsService } from '@services/ipc-services/settings.service';
{{ count(preview.counts.projects, 'project') }}, - {{ count(preview.counts.sources, 'source') }}, + {{ count(preview.counts.sources, 'source') + }}{{ inboxNote(preview.counts.inbox) }}, {{ count(preview.counts.assets, 'file') }} ({{ preview.counts.bytes / 1024 | number : '1.0-0' }} @@ -157,8 +158,9 @@ import { SettingsService } from '@services/ipc-services/settings.service';
- Chat history, the inbox, and UI preferences from this window's - local storage. It does not hold projects, sources, or files. + Chat history and UI preferences from this window's local + storage. It does not hold projects, sources, the inbox, or + files.
@@ -227,8 +229,15 @@ export class StorageSettingsComponent implements OnInit { this.form = formBuilder.group({}); } - count(n: number, noun: string) { - return `${n} ${noun}${n === 1 ? '' : 's'}`; + count(n: number, noun: string, plural = `${noun}s`) { + return `${n} ${n === 1 ? noun : plural}`; + } + + /** " (2 inbox entries)" when the backup has inbox entries. */ + inboxNote(inbox = 0) { + return inbox > 0 + ? ` (${this.count(inbox, 'inbox entry', 'inbox entries')})` + : ''; } ngOnInit() { @@ -295,7 +304,7 @@ export class StorageSettingsComponent implements OnInit { `Restored ${this.count(r.projects, 'project')}, ${this.count( r.sources, 'source' - )}, ` + + )}${this.inboxNote(r.inbox)}, ` + `and ${this.count(r.assets, 'file')}. ` + 'Knowledge reloads now.'; this.notifications.success( diff --git a/src/kc_angular/src/app/components/shared/session.component.ts b/src/kc_angular/src/app/components/shared/session.component.ts new file mode 100644 index 00000000..74d0a57d --- /dev/null +++ b/src/kc_angular/src/app/components/shared/session.component.ts @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2026 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, OnDestroy } from '@angular/core'; +import { asapScheduler, observeOn, Subscription } from 'rxjs'; +import { ConfirmationService } from 'primeng/api'; +import { + BackendService, + ConnectionState, +} from '@services/ipc-services/backend.service'; +import { StorageService } from '@services/ipc-services/storage.service'; + +const TITLES: Record = { + active: '', + none: 'Open your library', + expired: 'Session ended', + signedOut: 'Logged out', + unreachable: 'Storage service not responding', +}; + +/** + * Blocks the app while the connection is not active, so no change can look + * saved. Queued changes stay in this tab. The user starts a new session + * with a launch link, then the queued changes are saved in order. + */ +@Component({ + selector: 'app-session-dialog', + template: ` + +
+
+ {{ reason }} +
+
This page has no browser session.
+
+ You logged out. This tab shows no library data. +
+
{{ reason }}
+ +
+ {{ unsaved }} change{{ unsaved === 1 ? '' : 's' }} not saved yet. They + stay in this tab and are saved when you continue. +
+ + +
    +
  1. + In the terminal that runs yarn browser, press Enter + for a new link. +
    2. +
  2. Paste the link here, then select Continue.
    3. +
+ +
+ You can also open the link in a new tab, then select Check again + here. +
+ + +
+ {{ error }} +
+
+ + + + + + `, +}) +export class SessionDialogComponent implements OnDestroy { + state: ConnectionState = 'active'; + unsaved = 0; + link = ''; + error = ''; + busy = false; + private subscriptions: Subscription[] = []; + + constructor(public backend: BackendService, storage: StorageService) { + this.subscriptions.push( + backend.state.subscribe((state) => { + this.state = state; + this.error = ''; + }), + storage.unsaved + .pipe(observeOn(asapScheduler)) + .subscribe((count) => (this.unsaved = count)) + ); + } + + get title() { + return TITLES[this.state]; + } + + get reason() { + return this.backend.reason; + } + + ngOnDestroy() { + this.subscriptions.forEach((s) => s.unsubscribe()); + } + + async reconnect(useCookie = false) { + this.busy = true; + this.error = ''; + try { + await this.backend.reconnect(useCookie ? undefined : this.link); + this.link = ''; + // Nothing loaded yet (no session at start, or after logout): load now + if (!this.backend.libraryLoaded) { + location.reload(); + } + } catch (e) { + this.error = e instanceof Error ? e.message : String(e); + } finally { + this.busy = false; + } + } +} + +/** Title bar of the browser client: save state and logout. */ +@Component({ + selector: 'app-session-status', + template: ` +
+ {{ state === 'active' ? 'Saving…' : unsaved + ' not saved' }} + +
+ `, +}) +export class SessionStatusComponent implements OnDestroy { + state: ConnectionState = 'active'; + unsaved = 0; + private subscriptions: Subscription[] = []; + + constructor( + private backend: BackendService, + private storage: StorageService, + private confirm: ConfirmationService + ) { + this.subscriptions.push( + backend.state.subscribe((state) => (this.state = state)), + storage.unsaved + .pipe(observeOn(asapScheduler)) + .subscribe((count) => (this.unsaved = count)) + ); + } + + ngOnDestroy() { + this.subscriptions.forEach((s) => s.unsubscribe()); + } + + /** Save queued changes first. Ask before changes are lost. */ + async logout() { + const saved = await this.storage.flush(); + if (saved) { + return this.endSession(); + } + const count = this.storage.unsavedCount; + this.confirm.confirm({ + key: 'session-logout', + header: 'Changes not saved', + message: `${count} change${ + count === 1 ? ' is' : 's are' + } not saved. If you log out now, ${ + count === 1 ? 'it is' : 'they are' + } lost.`, + icon: 'pi pi-exclamation-triangle', + acceptLabel: 'Log out and discard', + rejectLabel: 'Stay', + acceptButtonStyleClass: 'p-button-danger p-button-text', + rejectButtonStyleClass: 'p-button-text', + accept: () => this.endSession(), + }); + } + + private async endSession() { + try { + await this.backend.logout(); + } catch (e) { + this.confirm.confirm({ + key: 'session-logout', + header: 'Logout failed', + message: e instanceof Error ? e.message : String(e), + rejectVisible: false, + acceptLabel: 'OK', + }); + return; + } + // Remove library data from this page, then load a page without it + this.storage.closeLibrary(); + location.replace(`${location.origin}/#signed-out`); + } +} diff --git a/src/kc_angular/src/app/components/source-components/ks-details.component.ts b/src/kc_angular/src/app/components/source-components/ks-details.component.ts index aa7a28f8..6245ea15 100755 --- a/src/kc_angular/src/app/components/source-components/ks-details.component.ts +++ b/src/kc_angular/src/app/components/source-components/ks-details.component.ts @@ -95,10 +95,12 @@ export class KsDetailsComponent { }, 5000); } + /** Show "Saved" only after the service saved the change. */ update($event: KnowledgeSource) { if ($event) { - this.command.update([$event]); - this.onSaved(); + this.command.update([$event]).then((saved) => { + if (saved) this.onSaved(); + }); } } } diff --git a/src/kc_angular/src/app/platform/settings-store.ts b/src/kc_angular/src/app/platform/settings-store.ts index 3ffd51e5..aa019093 100644 --- a/src/kc_angular/src/app/platform/settings-store.ts +++ b/src/kc_angular/src/app/platform/settings-store.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { Observable, ReplaySubject } from 'rxjs'; +import { Injectable } from '@angular/core'; +import { HttpClient, HttpErrorResponse } from '@angular/common/http'; +import { firstValueFrom, Observable, ReplaySubject } from 'rxjs'; +import type { PreferenceDocument } from '@contracts/storage'; +import { BackendService } from '@services/ipc-services/backend.service'; import { createDefaultSettings, SettingsModel, @@ -31,11 +35,20 @@ export abstract class SettingsStore { /** Request the current settings. */ abstract start(): void; + /** + * Resolves when the saved settings are loaded. The app waits for it at + * startup, so no page reads or saves defaults in place of saved values. + */ + ready(): Promise { + return Promise.resolve(); + } + /** Merge a partial settings object and store it. */ abstract save(partial: object): void; } /** Desktop: Electron keeps settings in knowledge.settings.json. */ +@Injectable() export class DesktopSettingsStore extends SettingsStore { private subject = new ReplaySubject(1); readonly changes = this.subject.asObservable(); @@ -63,7 +76,11 @@ export class DesktopSettingsStore extends SettingsStore { } } -const BROWSER_SETTINGS_KEY = 'kc-settings'; +/** Settings of earlier versions, in localStorage of one browser address. */ +const LEGACY_SETTINGS_KEY = 'kc-settings'; + +/** Preference document of the browser client in the storage service. */ +const BROWSER_SETTINGS_PREFERENCE = 'browser-settings'; function browserDefaults(): SettingsModel { return createDefaultSettings({ @@ -110,26 +127,41 @@ function merge(target: any, source: any): any { return out; } -/** Browser: settings are UI preferences in localStorage. */ +/** + * Browser: the storage service keeps the settings with the library, so + * they do not depend on the browser address. Nothing is emitted before the + * saved settings load, so an early save cannot replace them with defaults. + */ +@Injectable() export class BrowserSettingsStore extends SettingsStore { private subject = new ReplaySubject(1); readonly changes = this.subject.asObservable(); private current: SettingsModel = browserDefaults(); + private loaded = false; + private loading?: Promise; + private early: object[] = []; + private writing = false; + private dirty = false; + + constructor(private http: HttpClient, private backend: BackendService) { + super(); + } + + private get url() { + return `${this.backend.storage.url}/v1/preferences/${BROWSER_SETTINGS_PREFERENCE}`; + } defaults() { return Promise.resolve(browserDefaults()); } start() { - try { - const saved = JSON.parse( - localStorage.getItem(BROWSER_SETTINGS_KEY) ?? '{}' - ); - this.current = merge(browserDefaults(), saved); - } catch { - this.current = browserDefaults(); - } - this.subject.next(this.current); + this.ready(); + } + + override ready() { + this.loading ??= this.load(); + return this.loading; } /** @@ -137,17 +169,80 @@ export class BrowserSettingsStore extends SettingsStore { * Subscribers that save settings in response cannot start a loop. */ save(partial: object) { + if (!this.loaded) { + this.early.push(partial); + return; + } const next = merge(this.current, partial); - const json = JSON.stringify(next); - if (json === JSON.stringify(this.current)) { + if (JSON.stringify(next) === JSON.stringify(this.current)) { return; } this.current = next; + this.write(); + setTimeout(() => this.subject.next(this.current)); + } + + private async load() { + await this.backend.whenActive(); + let saved: object = {}; + let migrate = false; try { - localStorage.setItem(BROWSER_SETTINGS_KEY, json); + const { preference } = await firstValueFrom( + this.http.get<{ preference: PreferenceDocument }>(this.url) + ); + saved = preference.data; } catch (e) { - console.warn('[Settings]: not saved', e); + if (e instanceof HttpErrorResponse && e.status === 404) { + // First start of this library: save the settings document once, + // with the settings of an earlier version at this address, if any + saved = legacySettings(); + migrate = true; + } else { + console.warn('[Settings]: not loaded', e); + } } - setTimeout(() => this.subject.next(this.current)); + this.current = merge(browserDefaults(), saved); + for (const partial of this.early) { + this.current = merge(this.current, partial); + } + this.loaded = true; + if (migrate || this.early.length > 0) this.write(); + this.early = []; + this.subject.next(this.current); + } + + /** Write the newest settings. Waits while the connection is not active. */ + private async write() { + this.dirty = true; + if (this.writing) return; + this.writing = true; + try { + while (this.dirty) { + this.dirty = false; + await this.backend.whenActive(); + try { + await firstValueFrom(this.http.put(this.url, this.current)); + localStorage.removeItem(LEGACY_SETTINGS_KEY); + } catch (e) { + if (this.backend.isConnectionFailure(e)) { + this.backend.connectionLost(e); + this.dirty = true; + } else { + console.warn('[Settings]: not saved', e); + } + } + } + } finally { + this.writing = false; + } + } +} + +function legacySettings(): object { + try { + const saved = JSON.parse(localStorage.getItem(LEGACY_SETTINGS_KEY) ?? '{}'); + return saved && typeof saved === 'object' ? saved : {}; + } catch { + return {}; } } diff --git a/src/kc_angular/src/app/services/command-services/ks-command.service.ts b/src/kc_angular/src/app/services/command-services/ks-command.service.ts index 433a6b1b..eea0191a 100755 --- a/src/kc_angular/src/app/services/command-services/ks-command.service.ts +++ b/src/kc_angular/src/app/services/command-services/ks-command.service.ts @@ -57,15 +57,17 @@ export class KsCommandService { private platform: Platform ) {} - update(ksList: KnowledgeSource[], notify = true) { - this.data.sources.update(ksList).then(() => { - if (notify) { + /** Resolves true when the service saved every source. */ + update(ksList: KnowledgeSource[], notify = true): Promise { + return this.data.sources.update(ksList).then((saved) => { + if (saved && notify) { this.notifications.success( 'Source Command', `Source${ksList.length > 1 ? 's' : ''} Updated`, ksList.map((k) => k.title).join(', ') ); } + return saved; }); } diff --git a/src/kc_angular/src/app/services/factory-services/project.service.ts b/src/kc_angular/src/app/services/factory-services/project.service.ts index 079dd00f..67ebabb3 100755 --- a/src/kc_angular/src/app/services/factory-services/project.service.ts +++ b/src/kc_angular/src/app/services/factory-services/project.service.ts @@ -241,7 +241,8 @@ export class ProjectService { * e.g. create an update with ID (required) and an array removeKnowledgeSource[], etc. * @param updates An array of ProjectUpdateRequest objects */ - async updateProjects(updates: ProjectUpdateRequest[]) { + async updateProjects(updates: ProjectUpdateRequest[]): Promise { + let saved = true; for (const update of updates) { // Make sure the target project exists let target = this.projectSource.find( @@ -252,7 +253,7 @@ export class ProjectService { `Attempting to update non-existant project with ID:`, update.id.value ); - return; + return false; } // Description accumulator, used by each operation to describe what actions occured in a meaningful way. @@ -382,7 +383,22 @@ export class ProjectService { } // Persist project to storage system - await this.storageService.updateProject(target); + if (!(await this.storageService.updateProject(target))) { + saved = false; + // Show only the added sources that the service has in this project + const added = new Set( + (update.addKnowledgeSource ?? []) + .filter( + (ks) => + this.storageService.savedProjectOf(ks.id.value) !== + target!.id.value + ) + .map((ks) => ks.id.value) + ); + target.knowledgeSource = target.knowledgeSource.filter( + (ks) => !added.has(ks.id.value) + ); + } this.projectSource = this.projectSource.filter( (p) => p.id.value !== update.id.value @@ -391,6 +407,7 @@ export class ProjectService { } this.refreshTree(); + return saved; } setCurrentProject(id: string | null): void { diff --git a/src/kc_angular/src/app/services/ingest-services/drag-and-drop.service.ts b/src/kc_angular/src/app/services/ingest-services/drag-and-drop.service.ts index a70aa52c..3b0e71c0 100755 --- a/src/kc_angular/src/app/services/ingest-services/drag-and-drop.service.ts +++ b/src/kc_angular/src/app/services/ingest-services/drag-and-drop.service.ts @@ -322,8 +322,8 @@ export class DragAndDropService { // Create a list of updates to be applied const updates = this.moveSourceRequests([source], projectId); - this.projects.updateProjects(updates).then(() => { - this._sourceDrop.next(source.id.value); + this.projects.updateProjects(updates).then((saved) => { + if (saved) this._sourceDrop.next(source.id.value); }); } @@ -341,7 +341,8 @@ export class DragAndDropService { // Create a list of updates to be applied const updates = this.moveSourceRequests(sources, projectId); - this.projects.updateProjects(updates).then(() => { + this.projects.updateProjects(updates).then((saved) => { + if (!saved) return; for (const source of sources) { this._sourceDrop.next(source.id.value); } diff --git a/src/kc_angular/src/app/services/ingest-services/ingest.service.ts b/src/kc_angular/src/app/services/ingest-services/ingest.service.ts index 6f3b28c8..2f7c92cf 100755 --- a/src/kc_angular/src/app/services/ingest-services/ingest.service.ts +++ b/src/kc_angular/src/app/services/ingest-services/ingest.service.ts @@ -30,6 +30,7 @@ import { KsFactoryService } from '../factory-services/ks-factory.service'; import { NotificationsService } from '../user-services/notifications.service'; import { ProjectService } from '../factory-services/project.service'; import { SettingsService } from '../ipc-services/settings.service'; +import { StorageService } from '../ipc-services/storage.service'; @Injectable({ providedIn: 'root', @@ -49,18 +50,13 @@ export class IngestService implements OnDestroy { private ipc: ElectronIpcService, private projects: ProjectService, private sanitizer: DomSanitizer, - private notify: NotificationsService + private notify: NotificationsService, + private storage: StorageService ) { this.autoscanSubscribe(); this.extensionSubscribe(); this.loadInbox(); - - this.queue.subscribe((sources) => { - // Persist source list every time it is update. Restore it on app load. - const sourceString = JSON.stringify(sources); - localStorage.setItem('ingest-queue', sourceString); - }); } ngOnDestroy() { @@ -69,7 +65,41 @@ export class IngestService implements OnDestroy { } } + /** Add new sources to the inbox and save them on the service. */ enqueue(ksList: KnowledgeSource[]) { + const added = this.addToList(ksList); + if (added.length <= 0) { + return; + } + this.storage.saveInboxEntries(added); + this.notify.success( + 'IngestService', + 'Source Imported', + `Imported ${added.length} Source${added.length > 1 ? 's' : ''}.` + ); + } + + /** + * Move inbox entries to a project. The project write moves each entry on + * the service in one step. An entry leaves the inbox list only after that + * write is saved. Repeating a transfer cannot copy a source, because the + * source keeps its ID. + */ + async transfer(sources: KnowledgeSource[], projectId: string) { + const saved = await this.projects.updateProjects([ + { id: { value: projectId }, addKnowledgeSource: sources }, + ]); + const moved = sources.filter( + (ks) => this.storage.savedProjectOf(ks.id.value) === projectId + ); + for (const ks of moved) { + this.add(ks); + } + return saved && moved.length === sources.length; + } + + /** Show sources in the inbox list. Returns the sources that were new. */ + private addToList(ksList: KnowledgeSource[]): KnowledgeSource[] { let ksQueue = this._queue.value; const ksNext: KnowledgeSource[] = []; @@ -95,7 +125,7 @@ export class IngestService implements OnDestroy { } if (ksNext.length <= 0) { - return; + return []; } ksQueue = ksQueue.concat(ksNext); @@ -116,16 +146,7 @@ export class IngestService implements OnDestroy { }); this._queue.next(ksQueue); - - if (ksList.length <= 0) { - return; - } - - this.notify.success( - 'IngestService', - 'Source Imported', - `Imported ${ksNext.length} Source${ksNext.length > 1 ? 's' : ''}.` - ); + return ksNext; } add(ks: KnowledgeSource) { @@ -137,6 +158,7 @@ export class IngestService implements OnDestroy { remove(ks: KnowledgeSource) { this.finalize(ks, 'remove'); + this.storage.deleteInboxEntry(ks); this._queue.next( this._queue.value.filter((k) => k.id.value !== ks.id.value) ); @@ -153,19 +175,19 @@ export class IngestService implements OnDestroy { this.settings.show('import'); } - private loadInbox() { - const inbox = localStorage.getItem('ingest-queue'); - if (inbox) { - const sources = JSON.parse(inbox); - - for (const source of sources) { - source.icon = undefined; - } - - this.favicon.extractFromKsList(sources).then((updated) => { - this.enqueue(updated); - }); + /** Show the inbox from the service, then move an older local inbox. */ + private async loadInbox() { + const sources = [...this.storage.inbox]; + sources.push(...(await this.storage.migrateLocalInbox())); + if (sources.length === 0) { + return; + } + for (const source of sources) { + source.icon = undefined; } + // Keep the saved order. Show every saved entry, also ones with equal titles. + const shown = await this.favicon.extractFromKsList(sources); + this._queue.next([...this._queue.value, ...shown]); } /** diff --git a/src/kc_angular/src/app/services/ipc-services/backend.service.ts b/src/kc_angular/src/app/services/ipc-services/backend.service.ts index b7868dc5..f2286714 100644 --- a/src/kc_angular/src/app/services/ipc-services/backend.service.ts +++ b/src/kc_angular/src/app/services/ipc-services/backend.service.ts @@ -16,12 +16,15 @@ import { Injectable } from '@angular/core'; import { + HttpErrorResponse, HttpEvent, HttpHandler, HttpInterceptor, HttpRequest, + HttpResponse, } from '@angular/common/http'; -import { Observable } from 'rxjs'; +import { BehaviorSubject, filter, firstValueFrom, Observable, tap } from 'rxjs'; +import { SESSION_TTL_HEADER } from '@contracts/storage'; export interface BackendEndpoint { url?: string; @@ -35,15 +38,102 @@ export interface BackendInfo { } /** - * How this client reaches the library API (and chat) and authorizes its - * requests. One implementation per platform, chosen at startup. + * State of the connection to the library API. + * - active: requests can change the library. + * - none: the page has no browser session yet. + * - expired: the browser session ended or another tab replaced it. + * - signedOut: the user logged out in this tab. + * - unreachable: the storage service does not respond. + * In every state except active, writes wait in the StorageService queue. + */ +export type ConnectionState = + | 'active' + | 'none' + | 'expired' + | 'signedOut' + | 'unreachable'; + +/** + * How this client reaches the library API (and chat), authorizes its + * requests, and tracks whether it can still write. One implementation per + * platform, chosen at startup. */ export abstract class BackendService { + protected readonly _state = new BehaviorSubject('none'); + readonly state = this._state.asObservable(); + + /** Why the connection is not active. Shown to the user. */ + reason = ''; + + /** True after the library loaded at startup. */ + libraryLoaded = false; + + /** Browser sessions (launch link, logout). False on the desktop. */ + abstract readonly sessions: boolean; + abstract get chat(): BackendEndpoint; abstract get storage(): BackendEndpoint; abstract load(): Promise; /** Add the credentials that this request needs, if it goes to our servers. */ abstract authorize(req: HttpRequest): HttpRequest; + /** + * Make the connection active again. Browser: with a pasted launch link, + * or with the session cookie that another tab started. + */ + abstract reconnect(launchLink?: string): Promise; + /** End the browser session. */ + abstract logout(): Promise; + + get current(): ConnectionState { + return this._state.value; + } + + /** Resolve when the connection is active. */ + whenActive(): Promise { + return firstValueFrom(this.state.pipe(filter((s) => s === 'active'))); + } + + /** + * The service refused the request before it changed anything (401, 403), + * or the request did not reach it (status 0). A retry after reconnect is + * safe for idempotent requests. + */ + isConnectionFailure(e: unknown): e is HttpErrorResponse { + return e instanceof HttpErrorResponse && [0, 401, 403].includes(e.status); + } + + /** The service refused the request before it read it. Any retry is safe. */ + isAuthFailure(e: unknown): e is HttpErrorResponse { + return e instanceof HttpErrorResponse && [401, 403].includes(e.status); + } + + /** Called for each response of the library API. */ + onResponse(_res: HttpResponse): void { + return; + } + + /** Called for each failed request to the library API. */ + connectionLost(e: HttpErrorResponse) { + if (this.current !== 'active') return; + if (e.status === 0) { + this.set('unreachable', 'The storage service does not respond.'); + } else if (this.isAuthFailure(e)) { + this.authFailed(e.error?.error?.message ?? e.message); + } + } + + /** True if the request goes to the library API. */ + isLibraryRequest(url: string) { + const base = this.storage.url; + return !!base && url.startsWith(base + '/'); + } + + protected abstract authFailed(message: string): void; + + protected set(state: ConnectionState, reason = '') { + this.reason = reason; + if (this._state.value !== state) this._state.next(state); + } } /** @@ -52,6 +142,8 @@ export abstract class BackendService { */ @Injectable() export class DesktopBackendService extends BackendService { + readonly sessions = false; + private info: BackendInfo = { chat: { error: 'Backend information not loaded.' }, storage: { error: 'Backend information not loaded.' }, @@ -77,6 +169,8 @@ export class DesktopBackendService extends BackendService { console.error(`[Backend]: ${name}: ${endpoint.error}`); } } + if (this.info.storage.url) this.set('active'); + else this.set('unreachable', this.info.storage.error); } authorize(req: HttpRequest) { @@ -93,17 +187,46 @@ export class DesktopBackendService extends BackendService { } return req; } + + async reconnect() { + await this.load(); + const url = this.info.storage.url; + const ok = url && (await fetch(`${url}/health`).catch(() => undefined))?.ok; + if (!ok) { + this.set('unreachable', 'The storage service does not respond.'); + throw new Error('The storage service does not respond.'); + } + this.set('active'); + } + + async logout() { + throw new Error('The desktop app has no browser session.'); + } + + protected authFailed(message: string) { + this.set( + 'unreachable', + `The storage service refused the request: ${message}` + ); + } } +const LAUNCH_CODE = /^[A-Za-z0-9_-]{43}$/; + /** * Browser: the storage service serves this page, so the API is on the * same origin. A launch code in the URL fragment starts a session cookie. * State-changing requests carry the CSRF token, which stays in memory. + * The service reports the remaining session time on each response, so the + * page knows when the session ends without another request. */ @Injectable() export class BrowserBackendService extends BackendService { + readonly sessions = true; + private csrfToken = ''; private storageEndpoint: BackendEndpoint = { error: 'No browser session.' }; + private expiry?: ReturnType; get chat(): BackendEndpoint { return { error: 'Chat is available in the desktop app.' }; @@ -114,12 +237,69 @@ export class BrowserBackendService extends BackendService { } async load() { - const code = new URLSearchParams(location.hash.slice(1)).get('launch'); - if (code) { + const params = new URLSearchParams(location.hash.slice(1)); + const code = params.get('launch'); + const signedOut = params.has('signed-out'); + if (code || signedOut) { // Remove the one-time code from the address bar and history history.replaceState(null, '', location.pathname + location.search); } + try { + await this.startSession(code ?? undefined); + } catch (e) { + if (signedOut) this.set('signedOut', 'You logged out.'); + else this.set('none', e instanceof Error ? e.message : String(e)); + } + } + authorize(req: HttpRequest) { + const sameOrigin = + req.url.startsWith('/') || req.url.startsWith(location.origin + '/'); + if (sameOrigin && !['GET', 'HEAD'].includes(req.method) && this.csrfToken) { + return req.clone({ setHeaders: { 'X-Knowledge-CSRF': this.csrfToken } }); + } + return req; + } + + override onResponse(res: HttpResponse) { + const ttl = Number(res.headers.get(SESSION_TTL_HEADER)); + if (res.headers.has(SESSION_TTL_HEADER) && Number.isFinite(ttl)) { + this.expireAfter(ttl * 1000); + } + } + + async reconnect(launchLink?: string) { + const input = launchLink?.trim(); + let code: string | undefined; + if (input) { + code = LAUNCH_CODE.test(input) ? input : this.codeFromLink(input); + } + await this.startSession(code); + } + + async logout() { + const res = await fetch('/v1/session', { + method: 'DELETE', + headers: { 'X-Knowledge-CSRF': this.csrfToken }, + }); + // 401: the session already ended. The result is the same. + if (!res.ok && res.status !== 401) { + throw new Error(await this.errorMessage(res, 'Logout failed.')); + } + this.csrfToken = ''; + clearTimeout(this.expiry); + this.set('signedOut', 'You logged out.'); + } + + protected authFailed(message: string) { + this.set( + 'expired', + `The browser session ended or was replaced (${message})` + ); + } + + /** Exchange a launch code, or read the current session cookie. */ + private async startSession(code?: string) { const res = code ? await fetch('/v1/session', { method: 'POST', @@ -127,35 +307,62 @@ export class BrowserBackendService extends BackendService { body: JSON.stringify({ code }), }) : await fetch('/v1/session'); - - if (res.ok) { - this.csrfToken = (await res.json()).csrfToken; - this.storageEndpoint = { url: location.origin }; - } else { - const message = await res - .json() - .then((b) => b?.error?.message) - .catch(() => undefined); - this.storageEndpoint = { - error: - message ?? - 'No browser session. Open the launch link that "yarn browser" prints.', - }; + if (!res.ok) { + throw new Error( + await this.errorMessage( + res, + 'No browser session. Open the launch link that "yarn browser" prints.' + ) + ); } + const session = await res.json(); + this.csrfToken = session.csrfToken; + this.storageEndpoint = { url: location.origin }; + this.expireAfter(Date.parse(session.expiresAt) - Date.now()); + this.set('active'); } - authorize(req: HttpRequest) { - const sameOrigin = - req.url.startsWith('/') || req.url.startsWith(location.origin + '/'); - if (sameOrigin && !['GET', 'HEAD'].includes(req.method) && this.csrfToken) { - return req.clone({ setHeaders: { 'X-Knowledge-CSRF': this.csrfToken } }); + /** Mark the session expired when the service ends it. */ + private expireAfter(ms: number) { + clearTimeout(this.expiry); + // One second late, so the service has ended the session for certain + this.expiry = setTimeout(() => { + if (this.current === 'active') { + this.set( + 'expired', + 'The browser session ended after a period without activity.' + ); + } + }, Math.max(0, ms) + 1000); + } + + private codeFromLink(link: string): string { + let url: URL; + try { + url = new URL(link); + } catch { + throw new Error('Paste the complete launch link from the terminal.'); } - return req; + if (url.origin !== location.origin) { + throw new Error( + `This link opens ${url.origin}, but this tab uses ${location.origin}. ` + + 'Unsaved changes in this tab can only go to the service at its own address. ' + + 'Start "yarn browser" again with the same library to use the same address.' + ); + } + const code = new URLSearchParams(url.hash.slice(1)).get('launch'); + if (!code) { + throw new Error('The link has no launch code. Copy the complete link.'); + } + return code; } -} -export function backendServiceFactory(desktop: boolean): BackendService { - return desktop ? new DesktopBackendService() : new BrowserBackendService(); + private async errorMessage(res: Response, fallback: string) { + return res + .json() + .then((b) => b?.error?.message ?? fallback) + .catch(() => fallback); + } } @Injectable() @@ -166,23 +373,43 @@ export class BackendAuthInterceptor implements HttpInterceptor { req: HttpRequest, next: HttpHandler ): Observable> { - return next.handle(this.backend.authorize(req)); + const library = this.backend.isLibraryRequest(req.url); + return next.handle(this.backend.authorize(req)).pipe( + tap({ + next: (event) => { + if (library && event instanceof HttpResponse) { + this.backend.onResponse(event); + } + }, + error: (e) => { + if (library && e instanceof HttpErrorResponse) { + this.backend.connectionLost(e); + } + }, + }) + ); } } /** * Runs before the app starts: get this instance's server addresses, then - * load projects and sources from the storage service. + * load projects, sources, and the inbox from the storage service. Without + * an active connection the session dialog explains what to do. */ export function initializeBackend( backend: BackendService, - storage: { load(): Promise } + storage: { load(): Promise }, + settings: { ready(): Promise } ) { return async () => { await backend.load(); + if (backend.current !== 'active') return; try { await storage.load(); + await settings.ready(); + backend.libraryLoaded = true; } catch (e: any) { + if (backend.isConnectionFailure(e)) return; const reason = e?.error?.error?.message ?? e?.message ?? String(e); console.error('[Storage]: load failed', e); alert(`Knowledge could not load your library.\n\n${reason}`); diff --git a/src/kc_angular/src/app/services/ipc-services/storage.service.ts b/src/kc_angular/src/app/services/ipc-services/storage.service.ts index 74ca64fe..b39c0fa1 100755 --- a/src/kc_angular/src/app/services/ipc-services/storage.service.ts +++ b/src/kc_angular/src/app/services/ipc-services/storage.service.ts @@ -15,17 +15,21 @@ */ /* - * Projects and sources. The storage service owns them. This service keeps - * an in-memory copy for the UI and writes every change to the service in - * order. It does not use localStorage for projects or sources. + * Projects, sources, and the inbox. The storage service owns them. This + * service keeps an in-memory copy for the UI and writes every change to the + * service in order. + * + * Writes are idempotent (PUT or DELETE by ID). A write that fails because + * the session ended or the service does not respond stays in the queue, + * with every later write, until the connection is active again. * * localStorage keeps only UI state: the current project, chat history, - * favicon cache, the inbox, and preferences. + * favicon cache, and preferences. */ import { Injectable } from '@angular/core'; import { HttpClient, HttpErrorResponse } from '@angular/common/http'; -import { firstValueFrom } from 'rxjs'; +import { BehaviorSubject, firstValueFrom } from 'rxjs'; import { KcProject } from '@app/models/project.model'; import { KnowledgeSource } from '@app/models/knowledge.source.model'; import { AutoscanService } from '@services/ingest-services/autoscan.service'; @@ -41,6 +45,7 @@ import { import { ASSET_FILENAME_HEADER, ASSET_ORIGINAL_PATH_HEADER, + CREATE_ONLY_HEADER, } from '@contracts/storage'; import type { AssetRecord, @@ -53,6 +58,21 @@ import type { } from '@contracts/storage'; import { createBackup, restoreBackup, RestoreResult } from './backup'; +/** The inbox of earlier versions, in renderer localStorage. */ +export const LEGACY_INBOX_KEY = 'ingest-queue'; + +/** localStorage keys with library data. Logout removes them. */ +const LIBRARY_CACHE_KEYS = [ + 'current-project', + 'topic-service-all-topics', + LEGACY_INBOX_KEY, +]; + +interface WriteTask { + run: () => Promise; + done: (saved: boolean) => void; +} + @Injectable({ providedIn: 'root', }) @@ -61,11 +81,20 @@ export class StorageService { private db = window.localStorage; private projectList: KcProject[] = []; + /** Inbox entries from the service, in order. */ + inbox: KnowledgeSource[] = []; + /** JSON of the last record written to or read from the service, by ID. */ private synced = new Map(); - /** Writes run one at a time, in call order. */ - private queue: Promise = Promise.resolve(); + /** Writes not saved yet, in call order. They run one at a time. */ + private tasks: WriteTask[] = []; + private running = false; + private readonly _unsaved = new BehaviorSubject(0); + readonly unsaved = this._unsaved.asObservable(); + + /** Set when the page may close without a warning (after logout). */ + private closing = false; constructor( private http: HttpClient, @@ -73,7 +102,17 @@ export class StorageService { private native: NativeFiles, private autoscan: AutoscanService, private notifications: NotificationsService - ) {} + ) { + this.backend.state.subscribe((state) => { + if (state === 'active') this.pump(); + }); + window.addEventListener('beforeunload', (event) => { + if (this.tasks.length > 0 && !this.closing) { + event.preventDefault(); + event.returnValue = ''; + } + }); + } private get api() { const url = this.backend.storage.url; @@ -85,7 +124,7 @@ export class StorageService { return `${url}/v1`; } - /** Load all projects and sources. Runs once before the app starts. */ + /** Load all projects, sources, and inbox entries. Runs once at startup. */ async load() { const [{ projects }, { sources }] = await Promise.all([ firstValueFrom(this.http.get(`${this.api}/projects`)), @@ -96,6 +135,10 @@ export class StorageService { for (const record of sources) { this.synced.set(`s:${record.id}`, JSON.stringify(this.writable(record))); const ks = this.revive(recordToSource(record) as KnowledgeSource); + if (record.projectId === null) { + this.inbox.push(ks); + continue; + } const list = byProject.get(record.projectId) ?? []; list.push(ks); byProject.set(record.projectId, list); @@ -145,7 +188,18 @@ export class StorageService { return this.projectList; } - async saveProject(project: KcProject) { + get unsavedCount() { + return this.tasks.length; + } + + /** The project that the service has for a source. null is the inbox. */ + savedProjectOf(sourceId: string): string | null | undefined { + const json = this.synced.get(`s:${sourceId}`); + return json === undefined ? undefined : JSON.parse(json).projectId; + } + + /** Resolves true when saved, false when the service rejected the write. */ + async saveProject(project: KcProject): Promise { if (!this.projectList.find((p) => p.id.value === project.id.value)) { this.projectList.push(project); } @@ -158,7 +212,8 @@ export class StorageService { } } - async updateProject(project: KcProject) { + /** Resolves true when saved, false when the service rejected the write. */ + async updateProject(project: KcProject): Promise { return this.enqueue(() => this.write(project)); } @@ -180,9 +235,165 @@ export class StorageService { } } - private enqueue(task: () => Promise): Promise { - this.queue = this.queue.then(task).catch((e) => this.report(e)); - return this.queue; + /** Put new entries in the inbox on the service. */ + async saveInboxEntries(sources: KnowledgeSource[]): Promise { + const results = await Promise.all( + sources.map((ks) => this.enqueue(() => this.writeInboxEntry(ks))) + ); + return results.every(Boolean); + } + + /** Remove an entry from the inbox. Its managed file stays. */ + deleteInboxEntry(ks: KnowledgeSource): Promise { + this.db.removeItem(`icon-${ks.id.value}`); + this.db.removeItem(`chat-${ks.id.value}`); + return this.enqueue(() => + this.remove(`s:${ks.id.value}`, `${this.api}/sources/${ks.id.value}`) + ); + } + + private async writeInboxEntry(ks: KnowledgeSource) { + if (ks.ingestType === 'file' && !ks.assetId) { + await this.copyFile(ks); + } + const record = sourceToRecord(ks, null); + await this.put( + `s:${record.id}`, + `${this.api}/sources/${record.id}`, + record + ); + } + + /** + * Move an inbox that an earlier version kept in localStorage to the + * service. Create-only writes: an entry that the service already has + * (for example, one moved to a project) does not change. The key is + * removed only after every entry is saved, so a stopped run repeats + * safely. Returns the entries that were added. + */ + async migrateLocalInbox(): Promise { + const raw = this.db.getItem(LEGACY_INBOX_KEY); + if (!raw) return []; + let entries: unknown; + try { + entries = JSON.parse(raw); + } catch { + entries = undefined; + } + if (!Array.isArray(entries)) { + this.notifications.error( + 'Storage', + 'Inbox Not Moved', + 'The inbox of an earlier version is not readable. It stays in this browser.', + 'toast' + ); + return []; + } + + const added: KnowledgeSource[] = []; + let complete = true; + for (const ks of entries as KnowledgeSource[]) { + if (typeof ks?.id?.value !== 'string') continue; + const saved = await this.enqueue(async () => { + if (ks.ingestType === 'file' && !ks.assetId) { + await this.copyFile(ks); + } + const record = sourceToRecord(ks, null); + const body = this.writable(record); + try { + await firstValueFrom( + this.http.put(`${this.api}/sources/${record.id}`, body, { + headers: { [CREATE_ONLY_HEADER]: '*' }, + }) + ); + } catch (e) { + if (e instanceof HttpErrorResponse && e.status === 412) return; + throw e; + } + this.synced.set(`s:${record.id}`, JSON.stringify(body)); + added.push(this.revive(ks)); + }); + complete = complete && saved; + } + if (complete) this.db.removeItem(LEGACY_INBOX_KEY); + return added; + } + + /** + * Resolve true when every queued write is saved. Resolve false after the + * timeout, or when the connection is not active. + */ + async flush(timeoutMs = 10000): Promise { + const end = Date.now() + timeoutMs; + while ( + this.tasks.length > 0 && + this.backend.current === 'active' && + Date.now() < end + ) { + await new Promise((r) => setTimeout(r, 100)); + } + return this.tasks.length === 0; + } + + /** + * After logout: drop queued writes and the library data that this page + * keeps, so the page can close. UI preferences stay. + */ + closeLibrary() { + this.closing = true; + const dropped = this.tasks; + this.tasks = []; + this._unsaved.next(0); + for (const task of dropped) task.done(false); + this.projectList = []; + this.inbox = []; + this.synced.clear(); + for (const key of Object.keys(this.db)) { + if ( + key.startsWith('icon-') || + key.startsWith('chat-') || + LIBRARY_CACHE_KEYS.includes(key) + ) { + this.db.removeItem(key); + } + } + } + + /** Resolves true when saved, false when the service rejected the write. */ + private enqueue(run: () => Promise): Promise { + return new Promise((done) => { + this.tasks.push({ run, done }); + this._unsaved.next(this.tasks.length); + this.pump(); + }); + } + + /** Run queued writes in order while the connection is active. */ + private async pump() { + if (this.running) return; + this.running = true; + try { + while (this.tasks.length > 0 && this.backend.current === 'active') { + const task = this.tasks[0]; + let saved = true; + try { + await task.run(); + } catch (e) { + if (this.backend.isConnectionFailure(e)) { + // Keep this write and all later writes. Retry after reconnect. + this.backend.connectionLost(e); + break; + } + this.report(e); + saved = false; + } + this.tasks.shift(); + this._unsaved.next(this.tasks.length); + task.done(saved); + } + } finally { + this.running = false; + } } private report(e: unknown) { @@ -222,6 +433,9 @@ export class StorageService { * Upload the bytes of a selected file. The service returns the managed * asset. Desktop and browser use this same operation. The original path * is sent as information only, when the desktop knows it. + * + * The upload waits while the connection is not active. It repeats only + * after 401 or 403, which the service returns before it stores anything. */ async uploadFile( file: File, @@ -234,12 +448,19 @@ export class StorageService { if (originalPath) { headers[ASSET_ORIGINAL_PATH_HEADER] = encodeURIComponent(originalPath); } - const { asset } = await firstValueFrom( - this.http.post<{ asset: AssetRecord }>(`${this.api}/assets`, file, { - headers, - }) - ); - return asset; + for (;;) { + await this.backend.whenActive(); + try { + const { asset } = await firstValueFrom( + this.http.post<{ asset: AssetRecord }>(`${this.api}/assets`, file, { + headers, + }) + ); + return asset; + } catch (e) { + if (!this.backend.isAuthFailure(e)) throw e; + } + } } /** @@ -321,7 +542,12 @@ export class StorageService { async activateRestore(id: string) { const { restored } = await firstValueFrom( this.http.post<{ - restored: { projects: number; sources: number; assets: number }; + restored: { + projects: number; + sources: number; + inbox: number; + assets: number; + }; }>(`${this.api}/restores/${id}/activate`, null) ); return restored; diff --git a/src/kc_angular/src/app/services/user-services/data.service.ts b/src/kc_angular/src/app/services/user-services/data.service.ts index dee4ed6e..15cdcbc0 100755 --- a/src/kc_angular/src/app/services/user-services/data.service.ts +++ b/src/kc_angular/src/app/services/user-services/data.service.ts @@ -59,16 +59,22 @@ export class DataService implements OnDestroy { return (await this.storage.ksList()).filter((ks) => ids.has(ks.id.value)); }, - /** Sources live inside their project. The storage service persists both. */ - update: async (ksList: KnowledgeSource[]) => { + /** + * Sources live inside their project. The storage service persists both. + * Resolves true when every source is saved. + */ + update: async (ksList: KnowledgeSource[]): Promise => { + let saved = true; for (const ks of ksList) { - await this._projects.updateProjects([ + const ok = await this._projects.updateProjects([ { id: ks.associatedProject, updateKnowledgeSource: [ks], }, ]); + saved = saved && ok; } + return saved; }, count: this.__ksList.asObservable().pipe(map((sources) => sources.length)), From 883290e411a3c2968f25661ac466ede99ab001ab Mon Sep 17 00:00:00 2001 From: Rob Royce Date: Thu, 8 Oct 2026 09:44:08 -0700 Subject: [PATCH 3/5] fix(ui): browser platform boundary for Save as PDF and other actions Review finding 4 and the same problem in other visible actions: - Save as PDF goes through a WebsitePdf capability. The desktop uses the hidden-window printToPDF path; the browser disables the button and says why. The loading dialog no longer waits for a reply that never comes. - Website Preview opens a new tab in the browser (no embedded browser). - File Preview shows the library copy through ManagedFiles on both platforms (PDF, images, text) instead of the original local path. - Opening a file without a library copy explains that it needs the desktop app instead of waiting for IPC forever. - Chat is disabled in the context menu and hidden on cards in the browser; "Show in files" appears only where it works. - The browser makes no request to the external favicon service. - Website links open with noopener. - Fixed console errors: a missing icon cache entry logged an error, the fresh-start route had an undefined segment, and /favicon.ico was missing. Co-Authored-By: Claude Opus 5.5 --- src/kc_angular/src/app/app.module.ts | 9 +++ .../src/app/components/history.component.ts | 3 +- .../source-components/ks-actions.component.ts | 33 +++++++--- .../source-components/ks-preview.component.ts | 59 +++++++++++++++++- .../ks-viewport/file-view.component.ts | 6 +- src/kc_angular/src/app/platform/platform.ts | 5 ++ .../src/app/platform/website-pdf.ts | 62 +++++++++++++++++++ .../command-services/ks-command.service.ts | 21 ++++++- .../ks-context-menu.service.ts | 11 +++- .../ingest-services/extractor.service.ts | 61 ++++++++---------- .../ingest-services/favicon.service.ts | 9 ++- src/kc_angular/src/index.html | 1 + src/kc_shared/models/browser.view.model.ts | 2 + 13 files changed, 230 insertions(+), 52 deletions(-) create mode 100644 src/kc_angular/src/app/platform/website-pdf.ts diff --git a/src/kc_angular/src/app/app.module.ts b/src/kc_angular/src/app/app.module.ts index 711fafbd..915239ad 100755 --- a/src/kc_angular/src/app/app.module.ts +++ b/src/kc_angular/src/app/app.module.ts @@ -23,6 +23,11 @@ import { initializeBackend, } from '@services/ipc-services/backend.service'; import { isDesktop } from '@app/platform/platform'; +import { + BrowserWebsitePdf, + DesktopWebsitePdf, + WebsitePdf, +} from '@app/platform/website-pdf'; import { BrowserSettingsStore, DesktopSettingsStore, @@ -377,6 +382,10 @@ import { CarouselModule } from 'primeng/carousel'; provide: BackendService, useClass: DESKTOP ? DesktopBackendService : BrowserBackendService, }, + { + provide: WebsitePdf, + useClass: DESKTOP ? DesktopWebsitePdf : BrowserWebsitePdf, + }, { provide: SettingsStore, useClass: DESKTOP ? DesktopSettingsStore : BrowserSettingsStore, diff --git a/src/kc_angular/src/app/components/history.component.ts b/src/kc_angular/src/app/components/history.component.ts index 6f2fe781..2802b85f 100755 --- a/src/kc_angular/src/app/components/history.component.ts +++ b/src/kc_angular/src/app/components/history.component.ts @@ -201,7 +201,8 @@ export class HistoryComponent { this.router.navigate([ 'app', 'inbox', - this.projects.getCurrentProjectId()?.value, + // No project yet: the same placeholder as the default route + this.projects.getCurrentProjectId()?.value ?? 'undefined', ]); return; } diff --git a/src/kc_angular/src/app/components/source-components/ks-actions.component.ts b/src/kc_angular/src/app/components/source-components/ks-actions.component.ts index 651c54a3..8170a882 100755 --- a/src/kc_angular/src/app/components/source-components/ks-actions.component.ts +++ b/src/kc_angular/src/app/components/source-components/ks-actions.component.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Platform } from '@app/platform/platform'; +import { WebsitePdf } from '@app/platform/website-pdf'; import { Component, EventEmitter, Input, Output } from '@angular/core'; import { KnowledgeSource } from '@app/models/knowledge.source.model'; import { ExtractorService } from '@services/ingest-services/extractor.service'; @@ -50,22 +52,28 @@ import { ExtractorService } from '@services/ingest-services/extractor.service'; > - + > + +