diff --git a/.gitignore b/.gitignore index 5a3e46ad..9882c7e2 100755 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,9 @@ auth.json !.yarn/versions *.d.ts + +# Local development data profiles +.dev-profiles/ + +# End-to-end test output +e2e/.output/ diff --git a/.node-version b/.node-version new file mode 100644 index 00000000..df6ae337 --- /dev/null +++ b/.node-version @@ -0,0 +1 @@ +24.21.0 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..e9cdfe86 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,188 @@ +# Local Development + +This guide tells you how to build and start Knowledge on your computer. +You do not need AWS, an Apple developer account, or an AI API key. + +## Toolchain + +| Tool | Version | Notes | +| --- | --- | --- | +| macOS | 26.6 (arm64) | Verified. Linux and Windows are not verified. | +| Node.js | 24.21.0 | Verified. See the compatibility note below. | +| Yarn | 3.2.4 | The repository contains this version (`.yarnrc.yml` `yarnPath`). Any global `yarn` starts it. | +| Electron | 26.3.0 | Installed by Yarn. | + +No native compiler is necessary. The install does not build `canvas`. + +Compatibility note: Angular 14 officially supports Node.js 14.15+ and 16.10+ only. +Node.js 24 builds and runs this branch, but it is outside the official range. +Electron 26 is end-of-life. Upgrade these in a separate step. + +## Setup + +```sh +yarn install --immutable +yarn build-dev +yarn start +``` + +`yarn start-dev` does the build and the start in one command. + +## Instances and local servers + +Each instance starts its own chat server on a random loopback port. Each +server requires a random token. The renderer receives the address and the +token through IPC (`A2E:Backend:Info`). One instance cannot use the server of +another instance. + +Only one instance can use a profile at a time. A second start with the same +profile prints a message and exits. Different profiles can run at the same time. + +## Data profiles + +A development run never uses your normal Knowledge data. + +| Start condition | Data location | +| --- | --- | +| Unpackaged, `KC_PROFILE_DIR` not set | `/.dev-profiles/dev` | +| `KC_PROFILE_DIR=` | `` | +| `KC_PROFILE_DIR=system` | Normal per-user locations (see below) | +| Packaged build, `KC_PROFILE_DIR` not set | Normal per-user locations | + +A profile contains these directories: + +| Directory | Contents | +| --- | --- | +| `userData` | Chromium storage: local storage (projects, sources, chat history), cache, cookies | +| `data` | Source text cache (`storage/sources`), OpenAI key file, temporary files | +| `settings` | `knowledge.settings.json` | +| `downloads` | Default save location. The autoscan folder is `downloads/Knowledge`. | + +To start with an empty profile: + +```sh +KC_PROFILE_DIR=/tmp/kc-empty yarn start +``` + +To delete the default development profile, delete `.dev-profiles/`. +Git ignores this directory. + +Caution: `KC_PROFILE_DIR=system` uses your real data. Autoscan can move files +out of the autoscan folder. Make a backup first. + +### Normal per-user locations (macOS) + +| Data | Location | +| --- | --- | +| Chromium storage | `~/Library/Application Support/knowledge-canvas` | +| Settings | `~/Library/Preferences/Knowledge/knowledge.settings.json` | +| Source text cache, API key | `~/.Knowledge` | +| Autoscan folder | `~/Downloads/Knowledge` | + +Do not confuse `~/Library/Application Support/Knowledge`. macOS owns that +directory (`knowledgeC.db`). Knowledge does not use it. + +## Tests + +Unit tests (Node.js 22.18 or later): + +```sh +yarn test +``` + +These check backup and restore, and profile path isolation. + +End-to-end tests (macOS): + +```sh +yarn build-dev +yarn e2e +``` + +The end-to-end tests start the real app with new, empty profiles in +`e2e/.output//`. They never use your normal profile. They do these checks: + +- `e2e/workflow.e2e.mjs`: create a project, save a link (local fixture site), + import a PDF and a text file, annotate, search, restart, export a backup, + and restore it into a second profile. +- `e2e/isolation.e2e.mjs`: two profiles at the same time get different server + addresses and tokens. Each instance rejects the other instance's requests. + A second start of the same profile exits. + +Test files are in `e2e/fixtures/`. To delete old test output, delete +`e2e/.output/`. + +To explore the UI by hand, use the interactive driver: + +```sh +node e2e/drive.mjs /tmp/kc-explore 7777 +curl -s -X POST http://127.0.0.1:7777 -d '{"op":"launch"}' +curl -s -X POST http://127.0.0.1:7777 -d '{"op":"shot","name":"home"}' +curl -s -X POST http://127.0.0.1:7777 -d '{"op":"close"}' +``` + +## Current limits + +- Annotations are topics and metadata entries (tag and value). The app has + no notes editor. +- Search uses titles, topics, descriptions, and source type. It does not + search file content. +- A backup does not include imported files. +- File sources keep the absolute path of the original file. After a restore, + file sources work only if the original files are still at those paths. +- Signed and notarized releases are not verified. +- AI chat with a real API key is not verified. +- Only macOS is verified. + +## Optional services + +| Service | Default | How to enable | +| --- | --- | --- | +| AI chat (OpenAI) | Off | Open the Chat tab of a source. Enter a key in the dialog. | +| Update check | Off | Packaged build only. Set `KC_ENABLE_UPDATES=1`. The update feed (S3) does not exist now. | +| Browser extension server | Off | Settings > Import. Uses port 9000. | +| Autoscan | Off | Settings > Import. | + +Without an API key, all source management works. Chat, summaries, and +AI topics do not work. The app shows an API key dialog. + +The key file (`openai.encrypted`) uses a fixed passphrase. Treat it as +plain text. + +## Backup and restore + +Use Settings > Backup. + +A backup contains all renderer local storage: + +- Projects and their sources +- Source topics and metadata annotations +- Chat history +- The inbox +- Theme and table preferences + +A backup does not contain: + +- Settings (`knowledge.settings.json`) +- Imported files. File sources keep the absolute path of the original file. +- The source text cache. The app extracts the text again when necessary. +- The API key + +Restore merges the project list and replaces items that have the same ID. +The app reloads after a restore. Restore accepts only files that Settings > +Backup made (`format: knowledge-backup`, version 1). + +## Packaging + +```sh +CSC_IDENTITY_AUTO_DISCOVERY=false yarn run pack +``` + +Use `yarn run pack`. The command `yarn pack` is a Yarn built-in. It makes an +npm archive and does not start electron-builder. + +The result is an unsigned app in `dist/mac-arm64`. Notarization starts only +when `APPLEID`, `APPLEPWD`, and `TEAMID` are set. Signing needs a valid +Developer ID certificate. + +Do not use `yarn publish`. Its target S3 bucket does not exist. diff --git a/README.md b/README.md index aa9b43b8..91192a09 100755 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ Unleash the power of context with the built-in Chromium browser (just like Googl Knowledge application more efficiently. - [Feature Roadmap](https://github.com/KnowledgeCanvas/knowledge/wiki/Feature-Roadmap): A list of existing and planned features. +- [Local Development](DEVELOPMENT.md): Build and start Knowledge from source, with an isolated data profile. - [Developer Guide](https://github.com/KnowledgeCanvas/knowledge/wiki/Development): Interested in contributing to Knowledge? Check here first! - [Debug Guide](https://github.com/KnowledgeCanvas/knowledge/wiki/Debugging): Something not working? Check here for diff --git a/Resources/notarize.js b/Resources/notarize.js index 8b5d68a4..3e911a92 100644 --- a/Resources/notarize.js +++ b/Resources/notarize.js @@ -28,6 +28,13 @@ exports.default = async function notarizing(context) { const password = process.env.APPLEPWD; const teamId = process.env.TEAMID; + if (!appleId || !password || !teamId) { + console.log( + "Skipping notarization: set APPLEID, APPLEPWD and TEAMID to notarize." + ); + return; + } + return await notarize({ appBundleId: "com.knowledge.canvas.app", appPath: `${appOutDir}/${appName}.app`, diff --git a/e2e/drive.mjs b/e2e/drive.mjs new file mode 100644 index 00000000..21635b41 --- /dev/null +++ b/e2e/drive.mjs @@ -0,0 +1,97 @@ +/* + * Interactive driver for manual exploration. + * + * node e2e/drive.mjs [port] + * curl -s -X POST http://127.0.0.1:7777 -d '{"op":"launch"}' + * + * Operations: launch, close, eval {js}, main {js}, click {sel,nth}, + * dblclick {sel,nth}, fill {sel,value}, press {sel,key}, files {sel,paths}, + * text {sel}, shot {name}, wait {ms}. + * Screenshots and the app log go into the profile directory. + */ + +import http from "node:http"; +import path from "node:path"; +import { close, launch, shot } from "./lib.mjs"; + +const profile = process.argv[2] && path.resolve(process.argv[2]); +const port = Number(process.argv[3] || 7777); +if (!profile) { + console.error("Usage: node e2e/drive.mjs [port]"); + process.exit(1); +} + +let ctx = null; +async function page() { + if (!ctx) { + ctx = await launch(profile, path.join(profile, "app.log")); + } + return ctx.page; +} + +const ops = { + launch: async () => ({ url: (await page()).url() }), + close: async () => { + if (ctx) { + await close(ctx); + ctx = null; + } + return { closed: true }; + }, + eval: async ({ js }) => ({ value: await (await page()).evaluate(js) }), + main: async ({ js }) => { + await page(); + const fn = new Function("electron", `return (async () => { ${js} })()`); + return { value: await ctx.app.evaluate(fn) }; + }, + click: async ({ sel, nth = 0 }) => { + await (await page()).locator(sel).nth(nth).click({ timeout: 10000 }); + return { ok: true }; + }, + dblclick: async ({ sel, nth = 0 }) => { + await (await page()).locator(sel).nth(nth).dblclick({ timeout: 10000 }); + return { ok: true }; + }, + fill: async ({ sel, value, nth = 0 }) => { + await (await page()).locator(sel).nth(nth).fill(value, { timeout: 10000 }); + return { ok: true }; + }, + press: async ({ sel, key, nth = 0 }) => { + await (await page()).locator(sel).nth(nth).press(key); + return { ok: true }; + }, + files: async ({ sel, paths, nth = 0 }) => { + await (await page()).locator(sel).nth(nth).setInputFiles(paths); + return { ok: true }; + }, + text: async ({ sel = "body" }) => ({ + value: (await (await page()).locator(sel).first().innerText()).slice( + 0, + 4000 + ), + }), + shot: async ({ name }) => ({ file: await shot(await page(), profile, name) }), + wait: async ({ ms }) => { + await new Promise((r) => setTimeout(r, ms)); + return { ok: true }; + }, +}; + +http + .createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + try { + const cmd = JSON.parse(body || "{}"); + if (!ops[cmd.op]) throw new Error(`Unknown op: ${cmd.op}`); + res.end(JSON.stringify(await ops[cmd.op](cmd))); + } catch (e) { + res.statusCode = 500; + res.end( + JSON.stringify({ error: String(e?.message ?? e).slice(0, 2000) }) + ); + } + }) + .listen(port, "127.0.0.1", () => { + console.log(`Driver on http://127.0.0.1:${port} (profile ${profile})`); + }); diff --git a/e2e/fixtures/recovery-fixture.pdf b/e2e/fixtures/recovery-fixture.pdf new file mode 100644 index 00000000..efaea4c5 Binary files /dev/null and b/e2e/fixtures/recovery-fixture.pdf differ diff --git a/e2e/fixtures/recovery-note.txt b/e2e/fixtures/recovery-note.txt new file mode 100644 index 00000000..1bed69ac --- /dev/null +++ b/e2e/fixtures/recovery-note.txt @@ -0,0 +1,3 @@ +Knowledge recovery test note. +The quick brown fox studies photosynthesis and chlorophyll. +Unique marker: zebrafish-lattice-42 diff --git a/e2e/fixtures/site/index.html b/e2e/fixtures/site/index.html new file mode 100644 index 00000000..a89b551c --- /dev/null +++ b/e2e/fixtures/site/index.html @@ -0,0 +1,15 @@ + + + + + Knowledge Fixture Site + + + + +
+

Knowledge Fixture Site

+

Unique marker: heron-cascade-19

+
+ + diff --git a/e2e/isolation.e2e.mjs b/e2e/isolation.e2e.mjs new file mode 100644 index 00000000..22de334a --- /dev/null +++ b/e2e/isolation.e2e.mjs @@ -0,0 +1,96 @@ +/* + * Instance isolation. Two profiles run at the same time. Each renderer + * must reach only its own local server. A second launch of the same + * profile must exit. + * + * Run: yarn build-dev && yarn e2e + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import CryptoJS from "crypto-js"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { ELECTRON, REPO, appEnv, close, launch, newRun } from "./lib.mjs"; + +function backendInfo(page) { + return page.evaluate(() => window.api.invoke("A2E:Backend:Info")); +} + +function callChat(page, url, token) { + return page.evaluate( + async ({ url, token }) => { + const headers = token ? { Authorization: `Bearer ${token}` } : {}; + const res = await fetch(`${url}/api/key`, { headers }); + return { status: res.status, body: await res.text() }; + }, + { url, token } + ); +} + +test( + "two profiles get separate servers and cannot use each other's", + { timeout: 180000 }, + async () => { + const run = newRun("isolation"); + const profileA = path.join(run, "profile-a"); + const profileB = path.join(run, "profile-b"); + const log = path.join(run, "app.log"); + + // Profile A has an API key file. Profile B has none. + fs.mkdirSync(path.join(profileA, "data"), { recursive: true }); + fs.writeFileSync( + path.join(profileA, "data", "openai.encrypted"), + CryptoJS.AES.encrypt("sk-dummy-profile-a", "unsecured").toString() + ); + + const a = await launch(profileA, log); + const b = await launch(profileB, log); + try { + const infoA = await backendInfo(a.page); + const infoB = await backendInfo(b.page); + + assert.match(infoA.chat.url, /^http:\/\/127\.0\.0\.1:\d+$/); + assert.match(infoB.chat.url, /^http:\/\/127\.0\.0\.1:\d+$/); + assert.notEqual(infoA.chat.url, infoB.chat.url); + assert.notEqual(infoA.chat.token, infoB.chat.token); + + // Each renderer reaches its own server with its own token + const ownA = await callChat(a.page, infoA.chat.url, infoA.chat.token); + assert.equal(ownA.status, 200, ownA.body); + assert.equal(JSON.parse(ownA.body).apiKeySet, true); + const ownB = await callChat(b.page, infoB.chat.url, infoB.chat.token); + assert.equal(ownB.status, 200, ownB.body); + assert.equal(JSON.parse(ownB.body).apiKeySet, false); + + // B cannot use A's server: no token, or B's token, gives 401 + assert.equal((await callChat(b.page, infoA.chat.url)).status, 401); + assert.equal( + (await callChat(b.page, infoA.chat.url, infoB.chat.token)).status, + 401 + ); + + // A second launch of profile A exits and leaves the first instance running + const second = spawnSync(ELECTRON, [REPO], { + cwd: REPO, + env: { ...appEnv(), KC_PROFILE_DIR: profileA }, + encoding: "utf8", + timeout: 30000, + }); + assert.equal( + second.status, + 0, + `second instance did not exit: ${second.signal}` + ); + assert.match( + second.stdout + second.stderr, + /already open in another instance/ + ); + assert.equal(a.app.windows().length, 1); + } finally { + await close(b); + await close(a); + } + } +); diff --git a/e2e/lib.mjs b/e2e/lib.mjs new file mode 100644 index 00000000..35aae002 --- /dev/null +++ b/e2e/lib.mjs @@ -0,0 +1,144 @@ +/* + * Launch helpers for driving Knowledge with isolated data profiles. + * + * Every launch sets KC_PROFILE_DIR. These helpers never use the normal + * per-user profile. Output (profiles, logs, screenshots) goes to + * e2e/.output/, which Git ignores. + */ + +import { _electron as electron } from "playwright-core"; +import fs from "node:fs"; +import http from "node:http"; +import path from "node:path"; + +export const REPO = path.resolve(import.meta.dirname, ".."); +export const FIXTURES = path.join(REPO, "e2e", "fixtures"); + +export const ELECTRON = path.join( + REPO, + "node_modules/electron/dist/Electron.app/Contents/MacOS/Electron" +); + +/** + * Environment for the app. The Node test runner sets NODE_TEST_CONTEXT in + * child processes. Electron's Node must not see it, or startup stalls. + */ +export function appEnv() { + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + delete env.NODE_OPTIONS; + return env; +} + +/** Create an empty output directory for one test run. */ +export function newRun(name) { + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const dir = path.join(REPO, "e2e", ".output", `${name}-${stamp}`); + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +/** Launch the app with the given profile directory. */ +export async function launch(profileDir, logFile, env = {}) { + if (!profileDir) { + throw new Error("launch() needs an explicit profile directory"); + } + const log = fs.createWriteStream(logFile, { flags: "a" }); + const app = await electron.launch({ + executablePath: ELECTRON, + args: [REPO], + cwd: REPO, + env: { ...appEnv(), ...env, KC_PROFILE_DIR: profileDir }, + timeout: 60000, + }); + app.process().stdout?.on("data", (d) => log.write(`[main] ${d}`)); + app.process().stderr?.on("data", (d) => log.write(`[main:err] ${d}`)); + const page = await app.firstWindow(); + page.on("console", (m) => log.write(`[renderer:${m.type()}] ${m.text()}\n`)); + page.on("pageerror", (e) => log.write(`[renderer:pageerror] ${e.message}\n`)); + await page.waitForSelector("app-create button", { timeout: 30000 }); + return { app, page, log }; +} + +export async function close(ctx) { + await ctx.app.close(); + ctx.log.end(); +} + +export async function shot(page, dir, name) { + const file = path.join(dir, `${name}.png`); + await page.screenshot({ path: file }); + return file; +} + +/** Save every download from this window into dir. */ +export async function captureDownloads(app, dir) { + await app.evaluate(({ BrowserWindow }, target) => { + const session = BrowserWindow.getAllWindows()[0].webContents.session; + session.removeAllListeners("will-download"); + session.on("will-download", (_event, item) => { + item.setSavePath(`${target}/${item.getFilename()}`); + }); + }, dir); +} + +/** Wait until a file exists and has stopped growing. */ +export async function waitForFile(file, timeout = 15000) { + const end = Date.now() + timeout; + let last = -1; + while (Date.now() < end) { + if (fs.existsSync(file)) { + const size = fs.statSync(file).size; + if (size > 0 && size === last) { + return file; + } + last = size; + } + await new Promise((r) => setTimeout(r, 250)); + } + throw new Error(`File did not appear: ${file}`); +} + +/** Serve e2e/fixtures/site on an ephemeral loopback port. */ +export async function startFixtureSite() { + const root = path.join(FIXTURES, "site"); + const server = http.createServer((req, res) => { + const name = + path.basename(new URL(req.url, "http://x").pathname) || "index.html"; + const file = path.join(root, name); + if (!fs.existsSync(file)) { + res.statusCode = 404; + return res.end(); + } + res.setHeader("Content-Type", "text/html; charset=utf-8"); + fs.createReadStream(file).pipe(res); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address(); + return { + url: `http://127.0.0.1:${port}/index.html`, + close: () => server.close(), + }; +} + +/** Read projects and their sources as the renderer sees them. */ +export async function readLibrary(page) { + return page.evaluate(() => { + const ids = JSON.parse(localStorage.getItem("kc-projects") ?? "[]"); + return ids.map((id) => { + const p = JSON.parse(localStorage.getItem(id)); + return { + id, + name: p.name, + sources: p.knowledgeSource.map((k) => ({ + title: k.title, + type: k.ingestType, + topics: k.topics ?? [], + annotations: (k.meta ?? []) + .filter((m) => m.key === "annotation") + .map((m) => m.value), + })), + }; + }); + }); +} diff --git a/e2e/steps.mjs b/e2e/steps.mjs new file mode 100644 index 00000000..1d872248 --- /dev/null +++ b/e2e/steps.mjs @@ -0,0 +1,128 @@ +/* + * User-level steps for the Knowledge UI. Each step uses the visible + * controls a person would use. + */ + +const TIMEOUT = 15000; + +export async function createProject(page, name, description = "") { + await page.locator("button.create-project").click(); + const dialog = page + .locator(".p-dialog") + .filter({ hasText: "Create Project" }); + await dialog.locator("input").first().fill(name); + if (description) { + await dialog.locator("textarea").fill(description); + } + await dialog.locator("button", { hasText: "Create" }).click(); + await page + .locator("p-tree, .p-tree") + .getByText(name, { exact: true }) + .first() + .waitFor({ timeout: TIMEOUT }); +} + +/** Add a website through the Web Import dialog into the current project. */ +export async function addLinkToProject(page, url, expectedTitle) { + await page.locator("app-create button").nth(2).click(); + const input = page.locator("app-import-web input#linkInput"); + await input.fill(url); + await input.press("Enter"); + await page + .locator("app-import-web table input") + .first() + .waitFor({ timeout: TIMEOUT }); + if (expectedTitle) { + await page.waitForFunction( + (title) => + [...document.querySelectorAll("app-import-web table input")].some( + (i) => i.value === title + ), + expectedTitle, + { timeout: TIMEOUT } + ); + } + await page + .locator("app-import-web p-selectbutton .p-button", { hasText: "Project" }) + .click(); + await page.locator("app-import-web button", { hasText: "Import" }).click(); +} + +/** Import files into the Inbox, then move all Inbox items to the current project. */ +export async function importFilesToProject(page, files) { + await page.locator("app-create input[type=file]").setInputFiles(files); + for (const file of files) { + const name = file.split("/").pop(); + await page.getByText(name).first().waitFor({ timeout: TIMEOUT }); + } + await page.getByText("Import All").click(); + await page.locator("button", { hasText: "Import" }).first().click(); +} + +export async function openTable(page) { + await page.locator("i.pi-table").first().click(); + await page + .locator("app-ks-table, p-table") + .first() + .waitFor({ timeout: TIMEOUT }); +} + +export async function tableTitles(page) { + return page.locator("p-table tbody tr td:nth-child(3)").allInnerTexts(); +} + +export async function openSource(page, title) { + await page.getByText(title, { exact: true }).first().dblclick(); + await page + .locator(".p-dialog, app-source") + .getByText(title) + .first() + .waitFor({ timeout: TIMEOUT }); +} + +export async function closeDialog(page) { + await page.keyboard.press("Escape"); + await page.waitForTimeout(500); +} + +export async function addAnnotation(page, key, value) { + await page.locator('input[placeholder="Tag"]').fill(key); + const valueInput = page.locator('input[placeholder="Value"]'); + await valueInput.fill(value); + await valueInput.press("Enter"); +} + +export async function addTopic(page, topic) { + const input = page.locator( + 'input[placeholder="Start typing to add a topic..."]' + ); + await input.fill(topic); + await input.press("Enter"); +} + +export async function openDocumentTab(page) { + await page.getByText("Document", { exact: true }).click(); +} + +/** Type into the global search box and return the suggestion texts. */ +export async function search(page, query) { + const input = page.locator("input.p-autocomplete-input"); + await input.click(); + await input.fill(""); + await input.type(query, { delay: 30 }); + await page + .locator(".p-autocomplete-panel li") + .first() + .waitFor({ timeout: TIMEOUT }); + await page.waitForTimeout(500); + return page + .locator(".p-autocomplete-panel li") + .evaluateAll((items) => + items.map((i) => i.innerText.replace(/\s+/g, " ").trim()) + ); +} + +export async function openSettings(page, section) { + await page.locator("i.pi-cog").first().click(); + await page.locator(".p-menuitem-link", { hasText: section }).click(); +} diff --git a/e2e/workflow.e2e.mjs b/e2e/workflow.e2e.mjs new file mode 100644 index 00000000..40e08786 --- /dev/null +++ b/e2e/workflow.e2e.mjs @@ -0,0 +1,130 @@ +/* + * Basic local workflow on fresh, isolated profiles: + * create a project, save a link, import a PDF and a text file, annotate, + * search, restart, export a backup, and restore it into a second profile. + * + * Run: yarn build-dev && yarn e2e + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { + FIXTURES, + captureDownloads, + close, + launch, + newRun, + readLibrary, + shot, + startFixtureSite, + waitForFile, +} from "./lib.mjs"; +import * as ui from "./steps.mjs"; + +const PDF = path.join(FIXTURES, "recovery-fixture.pdf"); +const TXT = path.join(FIXTURES, "recovery-note.txt"); +const PROJECT = "Recovery Project"; +const SITE_TITLE = "Knowledge Fixture Site"; + +const expected = [ + { title: SITE_TITLE, type: "website" }, + { title: "recovery-fixture.pdf", type: "file" }, + { title: "recovery-note.txt", type: "file" }, +]; + +test( + "basic workflow persists across restart and restores from backup", + { timeout: 300000 }, + async () => { + const run = newRun("workflow"); + const profileA = path.join(run, "profile-a"); + const profileB = path.join(run, "profile-b"); + const log = path.join(run, "app.log"); + const site = await startFixtureSite(); + + try { + // 1-6: create, save, import, view, annotate, search + let ctx = await launch(profileA, log); + await ui.createProject(ctx.page, PROJECT, "Created by the e2e workflow."); + await ui.addLinkToProject(ctx.page, site.url, SITE_TITLE); + await ui.importFilesToProject(ctx.page, [PDF, TXT]); + + await ui.openTable(ctx.page); + assert.deepEqual( + (await ui.tableTitles(ctx.page)).sort(), + expected.map((e) => e.title).sort() + ); + + await ui.openSource(ctx.page, "recovery-note.txt"); + await ui.addAnnotation(ctx.page, "annotation", "mentions chlorophyll"); + await ui.addTopic(ctx.page, "recoverytopic"); + await ctx.page.waitForTimeout(1500); + await ui.closeDialog(ctx.page); + + await ui.openSource(ctx.page, "recovery-fixture.pdf"); + await ui.openDocumentTab(ctx.page); + await ctx.page.waitForTimeout(2000); + await shot(ctx.page, run, "a-pdf-document"); + await ui.closeDialog(ctx.page); + + const results = await ui.search(ctx.page, "Fixture Site"); + assert.ok( + results.some((r) => r.includes(SITE_TITLE)), + `search results: ${results}` + ); + await close(ctx); + + // 7-9: restart and confirm the data remains + ctx = await launch(profileA, log); + const library = await readLibrary(ctx.page); + assert.equal(library.length, 1); + assert.equal(library[0].name, PROJECT); + assert.deepEqual( + library[0].sources + .map(({ title, type }) => ({ title, type })) + .sort((a, b) => a.title.localeCompare(b.title)), + [...expected].sort((a, b) => a.title.localeCompare(b.title)) + ); + const note = library[0].sources.find( + (s) => s.title === "recovery-note.txt" + ); + assert.deepEqual(note.topics, ["recoverytopic"]); + assert.deepEqual(note.annotations, ["mentions chlorophyll"]); + + // 10: export a backup + const downloads = path.join(run, "downloads"); + await captureDownloads(ctx.app, downloads); + await ui.openSettings(ctx.page, "Backup"); + await ctx.page.locator("button", { hasText: "Export" }).click(); + const date = new Date().toISOString().slice(0, 10); + const backup = await waitForFile( + path.join(downloads, `knowledge-backup-${date}.json`) + ); + await close(ctx); + + // 11: restore into an empty second profile, then restart it + ctx = await launch(profileB, log); + assert.deepEqual(await readLibrary(ctx.page), []); + await ui.openSettings(ctx.page, "Backup"); + await ctx.page + .locator("app-storage-settings input[type=file]") + .setInputFiles(backup); + await ctx.page.waitForFunction( + () => + JSON.parse(localStorage.getItem("kc-projects") ?? "[]").length === 1, + null, + { timeout: 15000 } + ); + await close(ctx); + + ctx = await launch(profileB, log); + assert.deepEqual(await readLibrary(ctx.page), library); + await ui.openTable(ctx.page); + await shot(ctx.page, run, "b-restored-table"); + await close(ctx); + } finally { + site.close(); + } + } +); diff --git a/package.json b/package.json index 65d90bfd..d801993e 100644 --- a/package.json +++ b/package.json @@ -33,20 +33,26 @@ "publish-all": "electron-builder -mwl --x64 --arm64 --publish always", "release": "standard-version", "start": "electron .", + "start-dev": "yarn build-dev && electron .", "start-windows": "./node_modules/electron/dist/electron.exe . --platform win --arch x64", "start-debug": "electron . --inspect=5858", - "test": "echo \"Error: no test specified\" && exit 1", + "test": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test \"tests/*.test.mts\"", + "e2e": "node --test --test-concurrency=1 \"e2e/*.e2e.mjs\"", "watch-main": "cd src/kc_angular && yarn run watch-main", "watch-main-dev": "cd src/kc_angular && yarn run watch-main-dev", "watch-electron": "cd src/kc_electron && yarn run watch-electron", "watch-electron-dev": "cd src/kc_electron && yarn run watch-electron-dev" }, "packageManager": "yarn@3.2.4", + "dependenciesMeta": { + "canvas": { + "built": false + } + }, "dependencies": { "@adobe/css-tools": "^4.3.2", "async": "^3.2.5", "axios": "^1.6.5", - "canvas": "^2.11.2", "ejs": "^3.1.9", "express": "^4.18.2", "follow-redirects": "^1.15.5", @@ -85,6 +91,7 @@ "eslint-plugin-html": "^7.1.0", "loader-utils": "^3.2.1", "node-loader": "^2.0.0", + "playwright-core": "^1.63.0", "prettier": "^2.8.8", "standard-version": "^9.5.0", "worker-loader": "^3.0.8" diff --git a/src/kc_angular/src/app/app.component.html b/src/kc_angular/src/app/app.component.html index e812e471..87affd7b 100755 --- a/src/kc_angular/src/app/app.component.html +++ b/src/kc_angular/src/app/app.component.html @@ -81,7 +81,7 @@
@@ -113,7 +113,7 @@ Icon for the Knowledge application.
diff --git a/src/kc_angular/src/app/app.module.ts b/src/kc_angular/src/app/app.module.ts index 31b4eb4e..9f0a5a33 100755 --- a/src/kc_angular/src/app/app.module.ts +++ b/src/kc_angular/src/app/app.module.ts @@ -13,6 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { APP_INITIALIZER } from '@angular/core'; +import { HTTP_INTERCEPTORS } from '@angular/common/http'; +import { + BackendAuthInterceptor, + BackendService, + loadBackend, +} from '@services/ipc-services/backend.service'; import { A11yModule } from '@angular/cdk/a11y'; import { AccordionModule } from 'primeng/accordion'; import { AppComponent } from '@app/app.component'; @@ -322,6 +329,17 @@ import { CarouselModule } from 'primeng/carousel'; CarouselModule, ], providers: [ + { + provide: APP_INITIALIZER, + useFactory: loadBackend, + deps: [BackendService], + multi: true, + }, + { + provide: HTTP_INTERCEPTORS, + useClass: BackendAuthInterceptor, + multi: true, + }, ConfirmationService, DialogService, IngestService, diff --git a/src/kc_angular/src/app/components/chat-components/message-templates/quiz.message.ts b/src/kc_angular/src/app/components/chat-components/message-templates/quiz.message.ts index 7011f363..a942fb07 100644 --- a/src/kc_angular/src/app/components/chat-components/message-templates/quiz.message.ts +++ b/src/kc_angular/src/app/components/chat-components/message-templates/quiz.message.ts @@ -37,7 +37,7 @@ interface QuizQuestion {
diff --git a/src/kc_angular/src/app/components/home.component.ts b/src/kc_angular/src/app/components/home.component.ts index eb8d7b25..a895c25f 100755 --- a/src/kc_angular/src/app/components/home.component.ts +++ b/src/kc_angular/src/app/components/home.component.ts @@ -210,7 +210,7 @@ import { finalize, map, take, takeUntil, tap } from 'rxjs/operators'; >
-
Import/Export
+
Backup
- +
+ A backup holds projects, sources, annotations, chat history + and the inbox. It does not hold settings or imported files. +
+
@@ -78,64 +84,53 @@ export class StorageSettingsComponent { constructor( private storage: StorageService, private formBuilder: FormBuilder, - private notifications: NotificationsService + private notifications: NotificationsService, + private settings: SettingsService ) { this.form = formBuilder.group({}); } - async onExport() { + onExport() { this.exporting = true; - await this.storage.export(); - this.exporting = false; + try { + const keys = this.storage.export(this.settings.get().system?.appVersion); + this.notifications.success( + 'Backup', + 'Backup Exported', + `Saved ${keys} storage entries.` + ); + } catch (e) { + this.notifications.error('Backup', 'Export Failed', `${e}`); + } finally { + this.exporting = false; + } } onImport($event: any) { - const files: any[] = $event.target.files; - if (!files) { + const input: HTMLInputElement = $event.target; + const file = input.files?.[0]; + if (!file) { return; } - const file: File = files[0]; file .text() - .then((importFile) => { - const imported = JSON.parse(importFile); - if (imported?.projects) { - const projects = imported.projects; - const projectList: string[] = []; - for (const project of projects) { - projectList.push(project.id.value); - const id = project.id.value; - const pStr = JSON.stringify(project); - if (pStr) { - localStorage.setItem(id, pStr); - } - } + .then((text) => { + const result = this.storage.restore(text); + this.notifications.success( + 'Backup', + 'Backup Restored', + `${result.projectsAdded} projects added, ${result.projectsReplaced} replaced. Reloading...` + ); - const projectsStr = localStorage.getItem('kc-projects'); - if (projectsStr) { - const ids: string[] = JSON.parse(projectsStr); - if (ids) { - const nextIds = []; - for (const id of ids) { - nextIds.push(id); - } - for (const id of projectList) { - nextIds.push(id); - } - ids.concat(projectList); - const idStr = JSON.stringify(nextIds); - localStorage.setItem('kc-projects', idStr); - } - } - } + // Reload so every service reads the restored storage + setTimeout(() => window.location.reload(), 1500); }) - .catch(() => { - this.notifications.error( - 'Storage Settings', - 'File Error', - 'Unable to read file.' - ); + .catch((e: Error) => { + this.notifications.error('Backup', 'Restore Failed', e.message); + }) + .finally(() => { + input.value = ''; }); } } diff --git a/src/kc_angular/src/app/components/shared/web.import.component.ts b/src/kc_angular/src/app/components/shared/web.import.component.ts index 065fbef5..44ea0032 100644 --- a/src/kc_angular/src/app/components/shared/web.import.component.ts +++ b/src/kc_angular/src/app/components/shared/web.import.component.ts @@ -175,26 +175,28 @@ export class WebImportComponent { // HostListener for pasting text @HostListener('paste', ['$event']) onPaste(event: ClipboardEvent) { - const text = event.clipboardData?.getData('text'); + this.addUrls(event.clipboardData?.getData('text') ?? ''); + } + + onSubmit(value: string) { + this.addUrls(value); + } - // Get all URLs in text - const regex = /https?:\/\/[^\s]+/g; - const matches = text?.match(regex); + /** + * Create Sources from every URL in the text, then clear the input. + */ + private addUrls(text: string) { + const matches = text.match(/https?:\/\/[^\s]+/g); - // Add URLs to array if (matches) { - const urls: string[] = []; - matches.forEach((match) => { - // Remove commas from end of URLs - if (match.endsWith(',')) { - match = match.slice(0, -1); - } - urls.push(match); - }); + // Remove trailing commas from pasted lists + const urls = matches.map((match) => + match.endsWith(',') ? match.slice(0, -1) : match + ); this.createSource(urls); } - // Clear form by resetting URL field, making sure the value is updated + // Reset after the input event so the field clears setTimeout(() => { this.linkForm.setValue({ url: '', @@ -203,8 +205,6 @@ export class WebImportComponent { }); } - onSubmit(value: string) {} - createSource(urls: string[]) { const requests: KnowledgeSourceFactoryRequest = { ingestType: 'website', diff --git a/src/kc_angular/src/app/components/source-components/ks-preview.component.ts b/src/kc_angular/src/app/components/source-components/ks-preview.component.ts index 9ebd8390..5aad7a96 100755 --- a/src/kc_angular/src/app/components/source-components/ks-preview.component.ts +++ b/src/kc_angular/src/app/components/source-components/ks-preview.component.ts @@ -62,7 +62,7 @@ export interface KsPreviewInput {