From efc4dd52a629ecfb90f68469280037a74f7013ee Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 13:31:24 -0500 Subject: [PATCH 001/156] feat: add core and space migration packages --- bun.lock | 18 + packages/core/index.ts | 1 + packages/core/migrate/idempotent/sql.d.ts | 4 + .../migrate/incremental/000_provision.sql | 15 + packages/core/migrate/incremental/sql.d.ts | 4 + packages/core/migrate/migrate.ts | 347 +++++++++++ packages/core/package.json | 9 + packages/core/tsconfig.json | 4 + packages/space/index.ts | 8 + packages/space/migrate/bootstrap.ts | 163 +++++ .../space/migrate/idempotent/000_update.sql | 20 + .../space/migrate/idempotent/001_memory.sql | 581 ++++++++++++++++++ .../space/migrate/idempotent/002_search.sql | 329 ++++++++++ .../idempotent/003_embedding_queue.sql | 163 +++++ packages/space/migrate/idempotent/sql.d.ts | 4 + .../migrate/incremental/000_provision.sql | 15 + .../space/migrate/incremental/001_memory.sql | 48 ++ .../incremental/002_embedding_queue.sql | 22 + packages/space/migrate/incremental/sql.d.ts | 4 + packages/space/migrate/migrate.ts | 366 +++++++++++ packages/space/package.json | 9 + packages/space/slug.ts | 18 + packages/space/tsconfig.json | 4 + 23 files changed, 2156 insertions(+) create mode 100644 packages/core/index.ts create mode 100644 packages/core/migrate/idempotent/sql.d.ts create mode 100644 packages/core/migrate/incremental/000_provision.sql create mode 100644 packages/core/migrate/incremental/sql.d.ts create mode 100644 packages/core/migrate/migrate.ts create mode 100644 packages/core/package.json create mode 100644 packages/core/tsconfig.json create mode 100644 packages/space/index.ts create mode 100644 packages/space/migrate/bootstrap.ts create mode 100644 packages/space/migrate/idempotent/000_update.sql create mode 100644 packages/space/migrate/idempotent/001_memory.sql create mode 100644 packages/space/migrate/idempotent/002_search.sql create mode 100644 packages/space/migrate/idempotent/003_embedding_queue.sql create mode 100644 packages/space/migrate/idempotent/sql.d.ts create mode 100644 packages/space/migrate/incremental/000_provision.sql create mode 100644 packages/space/migrate/incremental/001_memory.sql create mode 100644 packages/space/migrate/incremental/002_embedding_queue.sql create mode 100644 packages/space/migrate/incremental/sql.d.ts create mode 100644 packages/space/migrate/migrate.ts create mode 100644 packages/space/package.json create mode 100644 packages/space/slug.ts create mode 100644 packages/space/tsconfig.json diff --git a/bun.lock b/bun.lock index 10ba56f..0302f5f 100644 --- a/bun.lock +++ b/bun.lock @@ -49,6 +49,13 @@ "typescript", ], }, + "packages/core": { + "name": "@memory.build/core", + "version": "0.2.5", + "dependencies": { + "@pydantic/logfire-node": "^0.13.1", + }, + }, "packages/docs-site": { "name": "@memory.build/docs-site", "version": "0.0.0", @@ -123,6 +130,13 @@ "zod": "^4.0.0", }, }, + "packages/space": { + "name": "@memory.build/space", + "version": "0.2.5", + "dependencies": { + "@pydantic/logfire-node": "^0.13.1", + }, + }, "packages/web": { "name": "@memory.build/web", "version": "0.1.17", @@ -369,6 +383,8 @@ "@memory.build/client": ["@memory.build/client@workspace:packages/client"], + "@memory.build/core": ["@memory.build/core@workspace:packages/core"], + "@memory.build/docs-site": ["@memory.build/docs-site@workspace:packages/docs-site"], "@memory.build/embedding": ["@memory.build/embedding@workspace:packages/embedding"], @@ -377,6 +393,8 @@ "@memory.build/protocol": ["@memory.build/protocol@workspace:packages/protocol"], + "@memory.build/space": ["@memory.build/space@workspace:packages/space"], + "@memory.build/web": ["@memory.build/web@workspace:packages/web"], "@memory.build/worker": ["@memory.build/worker@workspace:packages/worker"], diff --git a/packages/core/index.ts b/packages/core/index.ts new file mode 100644 index 0000000..8861a32 --- /dev/null +++ b/packages/core/index.ts @@ -0,0 +1 @@ +export { type MigrateCoreOptions, migrateCore } from "./migrate/migrate"; diff --git a/packages/core/migrate/idempotent/sql.d.ts b/packages/core/migrate/idempotent/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/core/migrate/idempotent/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/core/migrate/incremental/000_provision.sql b/packages/core/migrate/incremental/000_provision.sql new file mode 100644 index 0000000..479b55c --- /dev/null +++ b/packages/core/migrate/incremental/000_provision.sql @@ -0,0 +1,15 @@ +create schema core; + +create table core.version +( version text not null +, at timestamptz not null default now() +); + +create unique index version_singleton_idx on core.version ((true)); -- only ONE row allowed +insert into core.version (version) values ('0.0.0'); + +create table core.migration +( name text not null constraint migration_pkey primary key +, applied_at_version text not null +, applied_at timestamptz not null default pg_catalog.clock_timestamp() +); diff --git a/packages/core/migrate/incremental/sql.d.ts b/packages/core/migrate/incremental/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/core/migrate/incremental/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts new file mode 100644 index 0000000..954eaa0 --- /dev/null +++ b/packages/core/migrate/migrate.ts @@ -0,0 +1,347 @@ +import { createHash } from "node:crypto"; +import { info, reportError, span } from "@pydantic/logfire-node"; +import { SQL, semver } from "bun"; + +import provisionSql from "./incremental/000_provision.sql" with { + type: "text", +}; + +interface Incremental { + name: string; + sql: string; +} + +const incrementals: Incremental[] = []; + +interface Idempotent { + name: string; + sql: string; +} + +const idempotents: Idempotent[] = []; + +const CORE_SCHEMA = "core"; +const REQUIRED_EXTENSIONS = [ + { name: "citext", minVersion: "1.6" }, + { name: "ltree", minVersion: "1.3" }, + { name: "vector", minVersion: "0.8.2" }, + { name: "pg_textsearch", minVersion: "1.1.0" }, +] as const; + +export interface MigrateCoreOptions { + targetVersion: string; + statementTimeout?: string; + lockTimeout?: string; + transactionTimeout?: string; + idleInTransactionSessionTimeout?: string; +} + +interface NormalizedMigrateCoreOptions { + targetVersion: string; + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function migrateCore( + sql: SQL, + options: MigrateCoreOptions, +): Promise { + const opts = normalizeMigrateCoreOptions(options); + const attributes = migrateAttributes(opts); + + await span("core.migrate", { + attributes, + callback: async () => { + try { + if (!semver.satisfies(opts.targetVersion, "*")) { + throw new Error(`Invalid target version: "${opts.targetVersion}"`); + } + const [key1, key2] = advisoryLockKey("memory-core:schema:core"); + + await sql.begin(async (tx) => { + await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${opts.lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${opts.transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${opts.idleInTransactionSessionTimeout}, true)`; + const acquired = await span("core.migrate.acquire_lock", { + attributes, + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error("Unable to acquire lock for core migrations."); + } + + await ensurePostgresVersion(tx); + for (const extension of REQUIRED_EXTENSIONS) { + await span("core.migrate.ensure_extension", { + attributes: { + "db.extension": extension.name, + "db.extension_min_version": extension.minVersion, + }, + callback: () => + ensureExtension(tx, extension.name, extension.minVersion), + }); + } + + if (!(await doesCoreExist(tx))) { + await span("core.migrate.provision", { + attributes, + callback: () => provisionCore(tx), + }); + info("Core schema provisioned", attributes); + } + await span("core.migrate.run", { + attributes, + callback: () => runMigrations(tx, opts), + }); + }); + info("Core migrations completed", attributes); + } catch (error) { + reportError("Core migration failed", error as Error, attributes); + throw error; + } + }, + }); +} + +function migrateAttributes( + options: NormalizedMigrateCoreOptions, +): Record { + return { + "db.schema": CORE_SCHEMA, + "core.target_version": options.targetVersion, + "core.required_extensions": REQUIRED_EXTENSIONS.map( + (extension) => `${extension.name}@>=${extension.minVersion}`, + ), + "db.statement_timeout": options.statementTimeout, + "db.lock_timeout": options.lockTimeout, + "db.transaction_timeout": options.transactionTimeout, + "db.idle_in_transaction_session_timeout": + options.idleInTransactionSessionTimeout, + }; +} + +function normalizeMigrateCoreOptions( + options: MigrateCoreOptions, +): NormalizedMigrateCoreOptions { + return { + targetVersion: options.targetVersion, + statementTimeout: options.statementTimeout ?? "20s", + lockTimeout: options.lockTimeout ?? "5s", + transactionTimeout: options.transactionTimeout ?? "1min", + idleInTransactionSessionTimeout: + options.idleInTransactionSessionTimeout ?? "5s", + }; +} + +function advisoryLockKey(schema: string): [number, number] { + const digest = createHash("sha256").update(schema).digest(); + return [digest.readInt32BE(0), digest.readInt32BE(4)]; +} + +const MAX_LOCK_RETRIES = 5; +const BASE_DELAY_MS = 100; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function acquireAdvisoryLock( + tx: SQL, + key1: number, + key2: number, +): Promise { + let acquired = false; + for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { + const [result] = await tx` + select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired + `; + if (result.acquired) { + acquired = true; + break; + } + if (attempt < MAX_LOCK_RETRIES - 1) { + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } + return acquired; +} + +async function doesCoreExist(tx: SQL): Promise { + const [{ coreExists }] = await tx` + select exists + ( + select 1 + from pg_namespace n + where n.nspname = ${CORE_SCHEMA} + ) as "coreExists" + `; + return coreExists; +} + +async function provisionCore(tx: SQL): Promise { + await tx.unsafe(provisionSql); +} + +async function ensurePostgresVersion(tx: SQL): Promise { + const [{ server_version_num }] = await tx` + select current_setting('server_version_num')::int as server_version_num + `; + if (server_version_num < 180000) { + throw new Error( + `PostgreSQL version 18 or higher is required (found ${server_version_num})`, + ); + } +} + +async function ensureExtension( + tx: SQL, + name: string, + minVersion: string, +): Promise { + const [installed] = await tx` + select x.extversion, n.nspname + from pg_extension x + inner join pg_namespace n on (x.extnamespace = n.oid) + where x.extname = ${name} + `; + + if (installed) { + if ( + installed.nspname === "public" && + semver.order(installed.extversion, minVersion) >= 0 + ) { + return; + } + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required in the "public" schema (found ${installed.extversion} installed in "${installed.nspname}")`, + ); + } + + const [available] = await tx` + select default_version + from pg_available_extensions + where name = ${name} + `; + + if (!available || semver.order(available.default_version, minVersion) < 0) { + const found = available + ? `found ${available.default_version} available` + : "not available"; + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required (${found})`, + ); + } + + try { + await tx`create extension if not exists ${tx(name)} with schema public`; + } catch (error: unknown) { + if ( + error instanceof SQL.PostgresError && + error.errno === "23505" && + error.constraint === "pg_extension_name_index" + ) { + return; + } + throw error; + } +} + +async function runMigrations( + tx: SQL, + options: NormalizedMigrateCoreOptions, +): Promise { + await assertSchemaOwnership(tx); + + const [{ version: dbVersion }] = await tx` + select version from core.version + `; + const cmp = semver.order(options.targetVersion, dbVersion); + if (cmp < 0) { + throw new Error( + `Target version (${options.targetVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the server.", + ); + } + if (cmp === 0) { + info("Core migration skipped, version current", { + "db.schema": CORE_SCHEMA, + "core.version": dbVersion, + "core.target_version": options.targetVersion, + }); + return; + } + + const sorted1 = [...incrementals].sort((a, b) => + a.name.localeCompare(b.name), + ); + + for (const migration of sorted1) { + const [{ existing }] = await tx` + select exists + ( + select 1 + from core.migration + where name = ${migration.name} + ) as existing + `; + + if (existing) { + continue; + } + + await span("core.migrate.incremental", { + attributes: { + "db.schema": CORE_SCHEMA, + "core.migration": migration.name, + "core.migration_type": "incremental", + "core.target_version": options.targetVersion, + }, + callback: async () => { + await tx.unsafe(migration.sql); + await tx` + insert into core.migration (name, applied_at_version) + values (${migration.name}, ${options.targetVersion})`; + }, + }); + info("Core migration applied", { + "db.schema": CORE_SCHEMA, + "core.migration": migration.name, + "core.migration_type": "incremental", + "core.target_version": options.targetVersion, + }); + } + + const sorted2 = [...idempotents].sort((a, b) => a.name.localeCompare(b.name)); + + for (const migration of sorted2) { + await span("core.migrate.idempotent", { + attributes: { + "db.schema": CORE_SCHEMA, + "core.migration": migration.name, + "core.migration_type": "idempotent", + "core.target_version": options.targetVersion, + }, + callback: () => tx.unsafe(migration.sql), + }); + } + + await tx`update core.version set version = ${options.targetVersion}, at = now()`; +} + +async function assertSchemaOwnership(tx: SQL): Promise { + const [result] = await tx` + select + n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner + from pg_catalog.pg_namespace n + where n.nspname = ${CORE_SCHEMA} + `; + + if (!result?.is_owner) { + throw new Error( + "Only the owner of the core schema can run database migrations", + ); + } +} diff --git a/packages/core/package.json b/packages/core/package.json new file mode 100644 index 0000000..a38ca89 --- /dev/null +++ b/packages/core/package.json @@ -0,0 +1,9 @@ +{ + "name": "@memory.build/core", + "version": "0.2.5", + "private": true, + "type": "module", + "dependencies": { + "@pydantic/logfire-node": "^0.13.1" + } +} diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json new file mode 100644 index 0000000..23b1d27 --- /dev/null +++ b/packages/core/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["**/*.ts", "**/*.d.ts"] +} diff --git a/packages/space/index.ts b/packages/space/index.ts new file mode 100644 index 0000000..3a7ed38 --- /dev/null +++ b/packages/space/index.ts @@ -0,0 +1,8 @@ +export { bootstrapSpaceDatabase } from "./migrate/bootstrap"; +export { type MigrateSpaceOptions, migrateSpace } from "./migrate/migrate"; +export { + isValidSlug, + isValidSpaceSchema, + schemaToSlug, + slugToSchema, +} from "./slug"; diff --git a/packages/space/migrate/bootstrap.ts b/packages/space/migrate/bootstrap.ts new file mode 100644 index 0000000..cc73a12 --- /dev/null +++ b/packages/space/migrate/bootstrap.ts @@ -0,0 +1,163 @@ +import { info, reportError, span } from "@pydantic/logfire-node"; +import { SQL, semver } from "bun"; + +const REQUIRED_EXTENSIONS = [ + { name: "citext", minVersion: "1.6" }, + { name: "ltree", minVersion: "1.3" }, + { name: "vector", minVersion: "0.8.2" }, + { name: "pg_textsearch", minVersion: "1.1.0" }, +] as const; + +/** + * Prepare a physical database to host space schemas. + * + * This does not create or migrate an individual space. Spaces are still created + * on demand by migrateSpace(), which provisions a specific me_ schema. + */ +export async function bootstrapSpaceDatabase( + sql: SQL, + statementTimeout: string = "20s", + lockTimeout: string = "5s", + transactionTimeout: string = "30s", + idleInTransactionSessionTimeout: string = "30s", + shardId?: number, +): Promise { + const attributes = { + "db.shard": shardId, + "db.statement_timeout": statementTimeout, + "db.lock_timeout": lockTimeout, + "db.transaction_timeout": transactionTimeout, + "db.idle_in_transaction_session_timeout": idleInTransactionSessionTimeout, + "space.required_extensions": REQUIRED_EXTENSIONS.map( + (extension) => `${extension.name}@>=${extension.minVersion}`, + ), + }; + + await span("space.bootstrap", { + attributes, + callback: async () => { + try { + await sql.begin(async (tx) => { + if (shardId !== undefined) { + await tx.unsafe(`set local pgdog.shard to ${String(shardId)}`); + } + await ensurePostgresVersion(tx); + await span("space.bootstrap.acquire_lock", { + callback: () => acquireAdvisoryLock(tx), + }); + await tx`select set_config('statement_timeout', ${statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${idleInTransactionSessionTimeout}, true)`; + for (const extension of REQUIRED_EXTENSIONS) { + await span("space.bootstrap.ensure_extension", { + attributes: { + "db.extension": extension.name, + "db.extension_min_version": extension.minVersion, + }, + callback: () => + ensureExtension(tx, extension.name, extension.minVersion), + }); + } + }); + info("Space bootstrap completed", attributes); + } catch (error) { + reportError("Space bootstrap failed", error as Error, attributes); + throw error; + } + }, + }); +} + +const MAX_LOCK_RETRIES = 5; +const BASE_DELAY_MS = 100; +const BOOTSTRAP_LOCK_ID = 1982010637711; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function acquireAdvisoryLock(tx: SQL): Promise { + let acquired = false; + for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { + const [result] = await tx` + select pg_try_advisory_xact_lock(${BOOTSTRAP_LOCK_ID}) as acquired + `; + if (result.acquired) { + acquired = true; + break; + } + if (attempt < MAX_LOCK_RETRIES - 1) { + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } + + if (!acquired) { + throw new Error(`Failed to acquire advisory lock`); + } +} + +async function ensurePostgresVersion(tx: SQL): Promise { + const [{ server_version_num }] = await tx` + select current_setting('server_version_num')::int as server_version_num + `; + if (server_version_num < 180000) { + throw new Error( + `PostgreSQL version 18 or higher is required (found ${server_version_num})`, + ); + } +} + +async function ensureExtension( + tx: SQL, + name: string, + minVersion: string, +): Promise { + const [installed] = await tx` + select x.extversion, n.nspname + from pg_extension x + inner join pg_namespace n on (x.extnamespace = n.oid) + where x.extname = ${name} + `; + + if (installed) { + if ( + installed.nspname === "public" && + semver.order(installed.extversion, minVersion) >= 0 + ) { + return; + } + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required in the "public" schema (found ${installed.extversion} installed in "${installed.nspname}")`, + ); + } + + const [available] = await tx` + select default_version + from pg_available_extensions + where name = ${name} + `; + + if (!available || semver.order(available.default_version, minVersion) < 0) { + const found = available + ? `found ${available.default_version} available` + : "not available"; + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required (${found})`, + ); + } + + try { + await tx`create extension if not exists ${tx(name)} with schema public`; + } catch (error: unknown) { + // Ignore duplicate extension errors (race condition in concurrent calls) + if ( + error instanceof SQL.PostgresError && + error.errno === "23505" && + error.constraint === "pg_extension_name_index" + ) { + return; + } + throw error; + } +} diff --git a/packages/space/migrate/idempotent/000_update.sql b/packages/space/migrate/idempotent/000_update.sql new file mode 100644 index 0000000..a102d9e --- /dev/null +++ b/packages/space/migrate/idempotent/000_update.sql @@ -0,0 +1,20 @@ +-- generic trigger function to update updated_at timestamp +create or replace function {{schema}}.update_updated_at() +returns trigger +as $func$ +begin + new.updated_at = pg_catalog.now(); + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, pg_temp; + +create or replace trigger memory_before_update_trg +before update on {{schema}}.memory +for each row +execute function {{schema}}.memory_before_update(); + +create or replace trigger embedding_queue_before_update_trg +before update on {{schema}}.embedding_queue +for each row +execute function {{schema}}.embedding_queue_before_update(); diff --git a/packages/space/migrate/idempotent/001_memory.sql b/packages/space/migrate/idempotent/001_memory.sql new file mode 100644 index 0000000..6de72b3 --- /dev/null +++ b/packages/space/migrate/idempotent/001_memory.sql @@ -0,0 +1,581 @@ +------------------------------------------------------------------------------- +-- memory triggers +------------------------------------------------------------------------------- +create or replace function {{schema}}.memory_before_update() +returns trigger +as $func$ +begin + -- always update the timestamp + new.updated_at = pg_catalog.now(); + + -- content changed -> new embedding needs to be generated + if old.content is distinct from new.content + and old.embedding is not distinct from new.embedding + then + new.embedding = null; + new.embedding_version = old.embedding_version operator(pg_catalog.+) 1; + end if; + + return new; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp -- public required for pgvector's `is not distinct from` +; + +create or replace trigger memory_before_update_trg +before update on {{schema}}.memory +for each row +execute function {{schema}}.memory_before_update(); + +------------------------------------------------------------------------------- +-- get memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_memory +( _user_id uuid +, _id uuid default null +) +returns table +( id uuid +, tree ltree +, meta jsonb +, temporal tstzrange +, content text +, created_at timestamptz +, updated_at timestamptz +, has_embedding bool +) +as $func$ + select + m.id + , m.tree + , m.meta + , m.temporal + , m.content + , m.created_at + , m.updated_at + , m.embedding is not null + from {{schema}}.memory m + where m.id = _id + and {{schema}}.has_tree_access(_user_id, m.tree, 1) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- create memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_memory +( _user_id uuid +, _tree ltree +, _content text +, _id uuid default null +, _meta jsonb default '{}' +, _temporal tstzrange default null +) +returns uuid +as $func$ +begin + if not {{schema}}.has_tree_access(_user_id, _tree, 2) then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + insert into {{schema}}.memory + ( id + , tree + , meta + , temporal + , content + ) + values + ( coalesce(_id, uuidv7()) + , _tree + , coalesce(_meta, '{}'::jsonb) + , _temporal + , _content + ) + returning id into strict _id + ; + return _id; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- patch memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.patch_memory +( _user_id uuid +, _id uuid +, _patch jsonb +) +returns bool +as $func$ +declare + _src ltree; + _dst ltree; + _ok bool; +begin + -- at least one valid field must be present + select count(*) filter (where k in ('meta', 'tree', 'temporal', 'content')) > 0 + into strict _ok + from jsonb_each(_patch) o(k, v) + ; + + if not _ok then + raise exception 'no valid patch fields found' + using errcode = 'invalid_parameter_value'; + end if; + + _dst = (_patch->>'tree')::ltree; + + -- cannot set tree to null + if _patch ? 'tree' and _dst is null then + raise exception 'tree cannot be set to null' + using errcode = 'invalid_parameter_value'; + end if; + + -- find the existing memory and get it's tree + select m.tree into _src + from {{schema}}.memory m + where m.id = _id + for update -- don't let anyone "move" the memory while we're working on it + ; + + if not found then + return false; + end if; + + with a as materialized + ( + select a.tree_path, a.access + from {{schema}}.calc_tree_access(_user_id) a + ) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 2 + ) + and + ( + _dst is null + or _src @> _dst + or exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + ) + into strict _ok + ; + + if not _ok then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + update {{schema}}.memory m set + tree = case when _patch ? 'tree' then (_patch->>'tree')::ltree else m.tree end + , meta = case when _patch ? 'meta' then _patch->'meta' else m.meta end + , temporal = case when _patch ? 'temporal' then (_patch->>'temporal')::tstzrange else m.temporal end + , content = case when _patch ? 'content' then _patch->>'content' else m.content end + where id = _id + returning id into _id + ; + + return _id is not null; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- move tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.move_tree +( _user_id uuid +, _src ltree +, _dst ltree +, _dry_run bool default false +) +returns bigint +as $func$ +declare + _has_src bool; + _has_dst bool; + _moved bigint; +begin + -- must have read/write on _src + -- must have read/write on _dst + with a as materialized + ( + select a.tree_path, a.access + from {{schema}}.calc_tree_access(_user_id) a + ) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 2 + ) + , exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + into strict _has_src, _has_dst + ; + + if not _has_src then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if not _has_dst then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + with x as + ( + select m.id + from {{schema}}.memory m + where _src @> m.tree + ) + , u as + ( + update {{schema}}.memory m + set tree = + case + when nlevel(m.tree) = nlevel(_src) then _dst + else _dst || subpath(m.tree, nlevel(_src), nlevel(m.tree) - nlevel(_src)) + end + from x + where m.id = x.id + and not _dry_run + ) + select count(*) into strict _moved + from x + ; + return _moved; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- copy tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.copy_tree +( _user_id uuid +, _src ltree +, _dst ltree +, _dry_run bool default false +) +returns bigint +as $func$ +declare + _has_src bool; + _has_dst bool; + _copied bigint; +begin + -- must have read on _src + -- must have read/write on _dst + with a as materialized + ( + select a.tree_path, a.access + from {{schema}}.calc_tree_access(_user_id) a + ) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 1 + ) + , exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + into strict _has_src, _has_dst + ; + + if not _has_src then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if not _has_dst then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + with m as + ( + select m.* + from {{schema}}.memory m + where _src @> m.tree + ) + , i as + ( + insert into {{schema}}.memory + ( meta + , tree + , temporal + , content + , embedding + , embedding_version + ) + select + m.meta + , case + when nlevel(m.tree) = nlevel(_src) then _dst + else _dst || subpath(m.tree, nlevel(_src), nlevel(m.tree) - nlevel(_src)) + end as dst + , m.temporal + , m.content + , m.embedding + , m.embedding_version + from m + where not _dry_run + ) + select count(*) into strict _copied + from m + ; + + return _copied; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_memory +( _user_id uuid +, _id uuid +) +returns bool +as $func$ +declare + _tree ltree; +begin + select m.tree into _tree + from {{schema}}.memory m + where m.id = _id + for update + ; + + if not found then + return false; + end if; + + if not {{schema}}.has_tree_access(_user_id, _tree, 2) then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + delete from {{schema}}.memory + where id = _id + ; + return found; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_tree +( _user_id uuid +, _tree ltree +, _dry_run bool default false +) +returns bigint +as $func$ +declare + _has_access bool; + _deleted bigint; +begin + -- must have read/write on _tree + select exists + ( + select 1 + from {{schema}}.calc_tree_access(_user_id) a + where a.tree_path @> _tree + and a.access >= 2 + ) + into strict _has_access + ; + + if not _has_access then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if _dry_run then + select count(*) into strict _deleted + from {{schema}}.memory m + where _tree @> m.tree + ; + else + with d as + ( + delete from {{schema}}.memory m + where _tree @> m.tree + returning id + ) + select count(*) into strict _deleted + from d + ; + end if; + + return _deleted; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _user_id uuid +, _tree ltree +, _access int4 +) +returns bigint +as $func$ + with x as materialized + ( + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= _access + ) + select count(*) + from {{schema}}.memory m + where _tree @> m.tree + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _user_id uuid +, _query lquery +, _access int4 +) +returns bigint +as $func$ + with x as materialized + ( + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= _access + ) + select count(*) + from {{schema}}.memory m + where m.tree ~ _query + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _user_id uuid +, _query ltxtquery +, _access int4 +) +returns bigint +as $func$ + with x as materialized + ( + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= _access + ) + select count(*) + from {{schema}}.memory m + where m.tree @ _query + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_tree +( _user_id uuid +, _query lquery +) +returns table +( tree ltree +, count bigint +) +as $func$ + with a as materialized + ( + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= 1 + ) + , m as + ( + select distinct m.id, m.tree + from {{schema}}.memory m + where m.tree ~ _query + and exists + ( + select 1 + from a + where a.tree_path @> m.tree + ) + ) + select + subltree(m.tree, 0, i) as tree + , count(m.id) as count + from m + cross join lateral generate_series(1, nlevel(m.tree)) i + group by 1 + order by 1 +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/space/migrate/idempotent/002_search.sql b/packages/space/migrate/idempotent/002_search.sql new file mode 100644 index 0000000..2790f70 --- /dev/null +++ b/packages/space/migrate/idempotent/002_search.sql @@ -0,0 +1,329 @@ +------------------------------------------------------------------------------- +-- search_memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.search_memory +( _user_id uuid +, _bm25 bm25query default null +, _vec halfvec({{embedding_dimensions}}) default null +, _max_vec_dist float8 default null +, _ltree ltree default null +, _lquery lquery default null +, _ltxtquery ltxtquery default null +, _meta_contains jsonb default null +, _temporal_within tstzrange default null +, _temporal_overlaps tstzrange default null +, _temporal_before timestamptz default null +, _temporal_after timestamptz default null +, _regexp text default null +, _limit bigint default 10 +) +returns table +( id uuid +, meta jsonb +, tree ltree +, temporal tstzrange +, content text +, has_embedding bool +, created_at timestamptz +, updated_at timestamptz +, score float8 +) +as $func$ +declare + _filter_count int = 0; + _score text; + _filters text[] = '{}'::text; + _order_by text; + _sql text; +begin + -- _bm25 OR _vec but NOT BOTH + if _bm25 is not null and _vec is not null then + raise exception 'providing both _bm25 and _vec is not supported' + using errcode = 'invalid_parameter_value'; + end if; + + if _max_vec_dist is not null and _vec is null then + raise exception '_max_vec_dist provided but _vec was not provided' + using errcode = 'invalid_parameter_value'; + end if; + + -- min 1, max 1000, default 10 + _limit = greatest(least(coalesce(_limit, 10), 1000), 1); + + -- bm25 or semantic + -- score and order by + case + when _bm25 is not null then + _filter_count = _filter_count + 1; + -- <@> is negative bm25 score. smaller values means better match. order by this for index scans + -- negative score * -1 = score. higher score means better match + _score = format($sql$, (m.content <@> %L::bm25query) * -1 as score$sql$, _bm25); + _order_by = format($sql$order by m.content <@> %L::bm25query, m.id$sql$, _bm25); + when _vec is not null then + _filter_count = _filter_count + 1; + -- <=> is cosine distance. smaller distance means better match. order by this for index scans + -- distance * -1 = "score". higher score means better match + _score = format($sql$, (m.embedding <=> %L::halfvec({{embedding_dimensions}})) * -1 as score$sql$, _vec); + _order_by = format($sql$order by m.embedding <=> %L::halfvec({{embedding_dimensions}}), m.id$sql$, _vec); + _filters = array_append + ( _filters + , $sql$and m.embedding is not null$sql$ + ); + if _max_vec_dist is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and (m.embedding <=> %L::halfvec({{embedding_dimensions}})) <= %L::float8$sql$, _vec, _max_vec_dist) + ); + end if; + else + _score = $sql$, -1 as score$sql$; + _order_by = $sql$order by m.id; + end case; + + -- ltree + if _ltree is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::ltree @> m.tree$sql$, _ltree) + ); + end if; + + -- lquery + if _lquery is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.tree ~ %L::lquery$sql$, _lquery) + ); + end if; + + -- ltxtquery + if _ltxtquery is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.tree @ %L::ltxtquery$sql$, _ltxtquery) + ); + end if; + + -- meta_contains + if _meta_contains is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.meta @> %L::jsonb$sql$, _meta_contains) + ); + end if; + + -- temporal_within + if _temporal_within is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::tstzrange @> m.temporal$sql$, _temporal_within) + ); + end if; + + -- temporal_overlaps + if _temporal_overlaps is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::tstzrange && m.temporal$sql$, _temporal_overlaps) + ); + end if; + + -- temporal_before + if _temporal_before is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and tstzrange('-infinity'::timestamptz, %L::timestamptz, '[]') @> m.temporal$sql$, _temporal_before) + ); + end if; + + -- temporal_after + if _temporal_after is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and tstzrange(%L::timestamptz, 'infinity'::timestamptz, '[]') @> m.temporal$sql$, _temporal_after) + ); + end if; + + -- regexp + if _regexp is not null then + if _filter_count = 0 then + raise exception 'regexp must not be the only filter criteria' + using errcode = 'invalid_parameter_value'; + end if; + _filters = array_append + ( _filters + , format($sql$and m.content ~* %L::text$sql$, _regexp) + ); + end if; + + -- construct the query + _sql = format( + $sql$ + with x as materialized + ( + select a.tree_path + from {{schema}}.calc_tree_access($1) a + where a.access >= 1 + ) + select + m.id + , m.meta + , m.tree + , m.temporal + , m.content + , m.embedding is not null + , m.created_at + , m.updated_at + %s + from {{schema}}.memory m + where exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) + %s + %s + limit $2 + $sql$ + , _score + , coalesce + (( + select string_agg(x, E'\n ') + from unnest(_filters) x + ), '') + , _order_by + ); + + return query execute _sql using _user_id, _limit; +end; +$func$ language plpgsql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- hybrid_search_memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.hybrid_search_memory +( _user_id uuid +, _bm25 bm25query +, _vec halfvec({{embedding_dimensions}}) +, _max_vec_dist float8 default null +, _ltree ltree default null +, _lquery lquery default null +, _ltxtquery ltxtquery default null +, _meta_contains jsonb default null +, _temporal_within tstzrange default null +, _temporal_overlaps tstzrange default null +, _temporal_before timestamptz default null +, _temporal_after timestamptz default null +, _regexp text default null +, _k float8 default 60.0 +, _candidate_limit bigint default 30 +, _fulltext_weight float8 default 1.0 +, _semantic_weight float8 default 1.0 +, _limit bigint default 10 +) +returns table +( id uuid +, meta jsonb +, tree ltree +, temporal tstzrange +, content text +, has_embedding bool +, created_at timestamptz +, updated_at timestamptz +, score float8 +) +as $func$ +declare +begin + if _bm25 is null then + raise exception '_bm25 must not be null' + using errcode = 'invalid_parameter_value'; + end if; + + if _vec is null then + raise exception '_vec must not be null' + using errcode = 'invalid_parameter_value'; + end if; + + _k = greatest(coalesce(_k, 60.0), 0.0); + _limit = greatest(least(coalesce(_limit, 10), 1000), 1); + _candidate_limit = greatest + ( least(coalesce(_candidate_limit, 30), 1000) + , _limit + ); + _fulltext_weight = greatest(least(coalesce(_fulltext_weight, 1.0), 1.0), 0.0); + _semantic_weight = greatest(least(coalesce(_semantic_weight, 1.0), 1.0), 0.0); + + -- reciprocal rank fusion + return query + select + coalesce(x1.id, x2.id) as id + , coalesce(x1.meta, x2.meta) as meta + , coalesce(x1.tree, x2.tree) as tree + , coalesce(x1.temporal, x2.temporal) as temporal + , coalesce(x1.content, x2.content) as content + , coalesce(x1.has_embedding, x2.has_embedding) as has_embedding + , coalesce(x1.created_at, x2.created_at) as created_at + , coalesce(x1.updated_at, x2.updated_at) as updated_at + , coalesce(_fulltext_weight / (_k + x1.rank), 0.0) + + coalesce(_semantic_weight / (_k + x2.rank), 0.0) as score + from + ( + select + row_number() over (order by m.score desc, m.id) as rank + , m.* + from {{schema}}.search_memory + ( _user_id => _user_id + , _bm25 => _bm25 + , _ltree => _ltree + , _lquery => _lquery + , _ltxtquery => _ltxtquery + , _meta_contains => _meta_contains + , _temporal_within => _temporal_within + , _temporal_overlaps => _temporal_overlaps + , _temporal_before => _temporal_before + , _temporal_after => _temporal_after + , _regexp => _regexp + , _limit => _candidate_limit + ) m + ) x1 + full outer join + ( + select + row_number() over (order by m.score desc, m.id) as rank + , m.* + from {{schema}}.search_memory + ( _user_id => _user_id + , _vec => _vec + , _max_vec_dist => _max_vec_dist + , _ltree => _ltree + , _lquery => _lquery + , _ltxtquery => _ltxtquery + , _meta_contains => _meta_contains + , _temporal_within => _temporal_within + , _temporal_overlaps => _temporal_overlaps + , _temporal_before => _temporal_before + , _temporal_after => _temporal_after + , _regexp => _regexp + , _limit => _candidate_limit + ) m + ) x2 on (x1.id = x2.id) + order by score desc, id + limit _limit + ; +end; +$func$ language plpgsql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/space/migrate/idempotent/003_embedding_queue.sql b/packages/space/migrate/idempotent/003_embedding_queue.sql new file mode 100644 index 0000000..5aba816 --- /dev/null +++ b/packages/space/migrate/idempotent/003_embedding_queue.sql @@ -0,0 +1,163 @@ + +------------------------------------------------------------------------------- +-- enqueue_embedding +------------------------------------------------------------------------------- +create or replace function {{schema}}.enqueue_embedding() +returns trigger +as $func$ +begin + insert into {{schema}}.embedding_queue (memory_id, embedding_version) + values (new.id, new.embedding_version); + return new; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +create or replace trigger memory_enqueue_embedding_insert +after insert on {{schema}}.memory +for each row +when (new.embedding is null) -- it's possible to insert with an embedding +execute function {{schema}}.enqueue_embedding() +; + +create or replace trigger memory_enqueue_embedding_update +after update on {{schema}}.memory +for each row +when +( old.content is distinct from new.content + and new.embedding is null +) +execute function {{schema}}.enqueue_embedding() +; + +------------------------------------------------------------------------------- +-- claim_embedding_batch +------------------------------------------------------------------------------- +create or replace function {{schema}}.claim_embedding_batch +( _batch_size int default 10 +, _lock_duration interval default '5 minutes' +, _max_attempts int default 3 +) +returns table +( queue_id bigint +, memory_id uuid +, embedding_version int +, content text +) +as $func$ +declare + _rec record; + _mem record; + _claimed_count int = 0; +begin + -- bulk-cancel visible queue rows superseded by a newer row for the same memory + update {{schema}}.embedding_queue eq + set outcome = 'cancelled' + where eq.outcome is null + and eq.vt <= now() + and exists + ( + select 1 + from {{schema}}.embedding_queue newer + where newer.memory_id = eq.memory_id + and newer.embedding_version > eq.embedding_version + and newer.outcome is null + ); + + -- sweep: finalize exhausted rows orphaned by worker crash + -- (attempts reached max but outcome was never written back) + update {{schema}}.embedding_queue + set + outcome = 'failed' + , last_error = coalesce(last_error, 'exceeded max attempts') + where outcome is null + and vt <= now() + and attempts >= _max_attempts + ; + + for _rec in + ( + select + eq.id + , eq.memory_id + , eq.embedding_version + from {{schema}}.embedding_queue eq + where eq.outcome is null + and eq.vt <= now() + and eq.attempts < _max_attempts + order by eq.vt + for update skip locked + ) + loop + -- check memory still exists + current version + select m.content, m.embedding_version + into _mem + from {{schema}}.memory m + where m.id = _rec.memory_id + ; + + if not found or _mem.content is null then + -- memory deleted or empty → cancel queue row + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = _rec.id; + continue; + end if; + + if _rec.embedding_version != _mem.embedding_version then + -- stale version → cancel + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = _rec.id; + continue; + end if; + + -- claim this row + update {{schema}}.embedding_queue q set + vt = now() + _lock_duration + , attempts = q.attempts + 1 + where id = _rec.id; + + queue_id = _rec.id; + memory_id = _rec.memory_id; + embedding_version = _rec.embedding_version; + content = _mem.content; + return next; + + _claimed_count = _claimed_count + 1; + exit when _claimed_count >= _batch_size; + end loop; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- prune embedding queue +------------------------------------------------------------------------------- +-- prune terminal queue rows older than the retention window. +-- runs opportunistically from the worker on spaces that returned no +-- claimable work, so the queue table doesn't grow unbounded. +-- +-- relies on embedding_queue_archive_idx (created_at) where outcome is not null +-- from migration 005, so the no-op case is cheap. +create or replace function {{schema}}.prune_embedding_queue(_retention interval default '7 days') +returns bigint +as $func$ +declare + pruned bigint; +begin + delete from {{schema}}.embedding_queue + where outcome is not null + and created_at < now() - _retention + ; + get diagnostics pruned = row_count; + return pruned; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; diff --git a/packages/space/migrate/idempotent/sql.d.ts b/packages/space/migrate/idempotent/sql.d.ts new file mode 100644 index 0000000..89b092e --- /dev/null +++ b/packages/space/migrate/idempotent/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const content: string; + export default content; +} diff --git a/packages/space/migrate/incremental/000_provision.sql b/packages/space/migrate/incremental/000_provision.sql new file mode 100644 index 0000000..e98b9d9 --- /dev/null +++ b/packages/space/migrate/incremental/000_provision.sql @@ -0,0 +1,15 @@ +create schema {{schema}}; + +create table {{schema}}.version +( version text not null +, at timestamptz not null default now() +); + +create unique index version_singleton_idx on {{schema}}.version ((true)); -- only ONE row allowed +insert into {{schema}}.version (version) values ('0.0.0'); + +create table {{schema}}.migration +( name text not null constraint migration_pkey primary key +, applied_at_version text not null +, applied_at timestamptz not null default pg_catalog.clock_timestamp() +); diff --git a/packages/space/migrate/incremental/001_memory.sql b/packages/space/migrate/incremental/001_memory.sql new file mode 100644 index 0000000..c66eea7 --- /dev/null +++ b/packages/space/migrate/incremental/001_memory.sql @@ -0,0 +1,48 @@ +------------------------------------------------------------------------------- +-- memory +------------------------------------------------------------------------------- +create table {{schema}}.memory +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, meta jsonb not null default '{}' check (jsonb_typeof(meta) = 'object') +, tree ltree not null default ''::ltree +, temporal tstzrange +, content text not null +, embedding halfvec({{embedding_dimensions}}) +, embedding_version int4 not null default 1 +, created_at timestamptz not null default now() +, updated_at timestamptz +); + +-- index for faceted search +create index memory_meta_gin_idx on {{schema}}.memory using gin (meta); + +-- index for temporal search +create index memory_temporal_gist_idx on {{schema}}.memory using gist (temporal) where (temporal is not null); + +-- index for BM25 text search +create index memory_content_bm25_idx on {{schema}}.memory using bm25 (content) +with (text_config = {{bm25_text_config}}, k1 = {{bm25_k1}}, b = {{bm25_b}}); + +-- index for vector similarity search +create index memory_embedding_hnsw_idx on {{schema}}.memory using hnsw (embedding halfvec_cosine_ops) +with (m = {{hnsw_m}}, ef_construction = {{hnsw_ef_construction}}); + +-- index for hierarchical organization +create index memory_tree_gist_idx on {{schema}}.memory using gist (tree); + +/* +enforce consistent temporal range conventions: +- point-in-time events: lower = upper with inclusive bounds '[same,same]' +- time periods: lower < upper with inclusive-exclusive bounds '[start,end)' +*/ +alter table {{schema}}.memory add constraint temporal_bounds_convention check +( + temporal is null + or ( + -- point-in-time: both bounds equal and inclusive + (lower(temporal) = upper(temporal) and lower_inc(temporal) and upper_inc(temporal)) + or + -- time range: start before end, inclusive-exclusive + (lower(temporal) < upper(temporal) and lower_inc(temporal) and not upper_inc(temporal)) + ) +); diff --git a/packages/space/migrate/incremental/002_embedding_queue.sql b/packages/space/migrate/incremental/002_embedding_queue.sql new file mode 100644 index 0000000..468fe45 --- /dev/null +++ b/packages/space/migrate/incremental/002_embedding_queue.sql @@ -0,0 +1,22 @@ +------------------------------------------------------------------------------- +-- embedding queue +------------------------------------------------------------------------------- +-- per-space embedding queue table +create table {{schema}}.embedding_queue +( id bigint generated always as identity primary key +, memory_id uuid not null references {{schema}}.memory(id) on delete cascade +, embedding_version int not null +, vt timestamptz not null default now() +, outcome text check (outcome is null or outcome in ('completed', 'failed', 'cancelled')) +, attempts int not null default 0 +, last_error text +, created_at timestamptz not null default now() +, updated_at timestamptz +); + +-- index to find items to claim +create index embedding_queue_claim_idx on {{schema}}.embedding_queue (vt) where outcome is null; +-- index also used in finding items to claim. used to ensure there aren't any items for the same memory with a newer version +create index embedding_queue_memory_idx on {{schema}}.embedding_queue (memory_id, embedding_version desc) where outcome is null; +-- index to find items that have resolved to an outcome. these can be pruned +create index embedding_queue_archive_idx on {{schema}}.embedding_queue (created_at) where outcome is not null; diff --git a/packages/space/migrate/incremental/sql.d.ts b/packages/space/migrate/incremental/sql.d.ts new file mode 100644 index 0000000..89b092e --- /dev/null +++ b/packages/space/migrate/incremental/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const content: string; + export default content; +} diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts new file mode 100644 index 0000000..70fa7fd --- /dev/null +++ b/packages/space/migrate/migrate.ts @@ -0,0 +1,366 @@ +import { createHash } from "node:crypto"; +import { info, reportError, span } from "@pydantic/logfire-node"; +import type { SQL } from "bun"; +import { semver } from "bun"; +import { isValidSlug, slugToSchema } from "../slug"; + +import provisionSql from "./incremental/000_provision.sql" with { + type: "text", +}; +import incremental001 from "./incremental/001_memory.sql" with { type: "text" }; +import incremental002 from "./incremental/002_embedding_queue.sql" with { + type: "text", +}; + +interface Incremental { + name: string; + sql: string; +} + +const incrementals: Incremental[] = [ + { name: "001_memory", sql: incremental001 }, + { name: "002_embedding_queue", sql: incremental002 }, +]; + +import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; +import idempotent001 from "./idempotent/001_memory.sql" with { type: "text" }; +import idempotent002 from "./idempotent/002_search.sql" with { type: "text" }; +import idempotent003 from "./idempotent/003_embedding_queue.sql" with { + type: "text", +}; + +interface Idempotent { + name: string; + sql: string; +} + +const idempotents: Idempotent[] = [ + { name: "000_update", sql: idempotent000 }, + { name: "001_memory", sql: idempotent001 }, + { name: "002_search", sql: idempotent002 }, + { name: "003_embedding_queue", sql: idempotent003 }, +]; + +export interface MigrateSpaceOptions { + slug: string; + targetVersion: string; + shardId?: number; + embeddingDimensions?: number; + bm25TextConfig?: string; + bm25K1?: number; + bm25B?: number; + hnswM?: number; + hnswEfConstruction?: number; + statementTimeout?: string; + lockTimeout?: string; + transactionTimeout?: string; + idleInTransactionSessionTimeout?: string; +} + +interface NormalizedMigrateSpaceOptions { + slug: string; + targetVersion: string; + shardId?: number; + embeddingDimensions: number; + bm25TextConfig: string; + bm25K1: number; + bm25B: number; + hnswM: number; + hnswEfConstruction: number; + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function migrateSpace( + sql: SQL, + options: MigrateSpaceOptions, +): Promise { + const opts = normalizeMigrateSpaceOptions(options); + const attributes = migrateAttributes(opts); + + await span("space.migrate", { + attributes, + callback: async () => { + try { + if (!isValidSlug(opts.slug)) { + throw new Error( + `Invalid space slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, + ); + } + if (!semver.satisfies(opts.targetVersion, "*")) { + throw new Error(`Invalid target version: "${opts.targetVersion}"`); + } + const schema = slugToSchema(opts.slug); + const schemaAttributes = { ...attributes, "db.schema": schema }; + const [key1, key2] = advisoryLockKey(`memory-space:schema:${schema}`); + + await sql.begin(async (tx) => { + if (opts.shardId !== undefined) { + if (!Number.isSafeInteger(opts.shardId)) { + throw new Error( + `shardId must be a safe integer, got: ${opts.shardId}`, + ); + } + await tx.unsafe(`set local pgdog.shard to ${String(opts.shardId)}`); + } + await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${opts.lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${opts.transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${opts.idleInTransactionSessionTimeout}, true)`; + const acquired = await span("space.migrate.acquire_lock", { + attributes: schemaAttributes, + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error( + `Unable to acquire lock for space slug ${opts.slug} migrations.`, + ); + } + + if (!(await doesSpaceExist(tx, schema))) { + await span("space.migrate.provision", { + attributes: schemaAttributes, + callback: () => provisionSpace(tx, schema), + }); + info("Space schema provisioned", schemaAttributes); + } + await span("space.migrate.run", { + attributes: schemaAttributes, + callback: () => runMigrations(tx, schema, opts), + }); + }); + info("Space migrations completed", schemaAttributes); + } catch (error) { + reportError("Space migration failed", error as Error, attributes); + throw error; + } + }, + }); +} + +function migrateAttributes( + options: NormalizedMigrateSpaceOptions, +): Record { + return { + "space.slug": options.slug, + "space.target_version": options.targetVersion, + "db.shard": options.shardId, + "db.statement_timeout": options.statementTimeout, + "db.lock_timeout": options.lockTimeout, + "db.transaction_timeout": options.transactionTimeout, + "db.idle_in_transaction_session_timeout": + options.idleInTransactionSessionTimeout, + }; +} + +function normalizeMigrateSpaceOptions( + options: MigrateSpaceOptions, +): NormalizedMigrateSpaceOptions { + return { + slug: options.slug, + targetVersion: options.targetVersion, + shardId: options.shardId, + embeddingDimensions: options.embeddingDimensions ?? 1536, + bm25TextConfig: options.bm25TextConfig ?? "english", + bm25K1: options.bm25K1 ?? 1.2, + bm25B: options.bm25B ?? 0.75, + hnswM: options.hnswM ?? 16, + hnswEfConstruction: options.hnswEfConstruction ?? 64, + statementTimeout: options.statementTimeout ?? "20s", + lockTimeout: options.lockTimeout ?? "5s", + transactionTimeout: options.transactionTimeout ?? "1min", + idleInTransactionSessionTimeout: + options.idleInTransactionSessionTimeout ?? "5s", + }; +} + +function templateVars( + schema: string, + options: NormalizedMigrateSpaceOptions, +): Record { + return { + ...options, + schema, + embedding_dimensions: options.embeddingDimensions, + bm25_text_config: options.bm25TextConfig, + bm25_k1: options.bm25K1, + bm25_b: options.bm25B, + hnsw_m: options.hnswM, + hnsw_ef_construction: options.hnswEfConstruction, + }; +} + +function advisoryLockKey(schema: string): [number, number] { + const digest = createHash("sha256").update(schema).digest(); + return [digest.readInt32BE(0), digest.readInt32BE(4)]; +} + +const MAX_LOCK_RETRIES = 5; +const BASE_DELAY_MS = 100; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function acquireAdvisoryLock( + tx: SQL, + key1: number, + key2: number, +): Promise { + let acquired = false; + for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { + const [result] = await tx` + select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired + `; + if (result.acquired) { + acquired = true; + break; + } + if (attempt < MAX_LOCK_RETRIES - 1) { + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } + return acquired; +} + +async function doesSpaceExist(tx: SQL, schema: string): Promise { + const [{ spaceExists }] = await tx` + select exists + ( + select 1 + from pg_namespace n + where n.nspname = ${schema} + ) as "spaceExists" + `; + return spaceExists; +} + +async function provisionSpace(tx: SQL, schema: string): Promise { + await tx.unsafe(template(provisionSql, { schema })); +} + +async function runMigrations( + tx: SQL, + schema: string, + options: NormalizedMigrateSpaceOptions, +): Promise { + // check ownership + await assertSchemaOwnership(tx, schema); + + // check version + const [{ version: dbVersion }] = await tx` + select version from ${tx(schema)}.version + `; + const cmp = semver.order(options.targetVersion, dbVersion); + // abort if target is older than the database + if (cmp < 0) { + throw new Error( + `Target version (${options.targetVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the server.", + ); + } + if (cmp === 0) { + // version matches. no need to run migrations + info("Space migration skipped, version current", { + "db.schema": schema, + "space.version": dbVersion, + "space.target_version": options.targetVersion, + }); + return; + } + + // run incremental migrations + const sorted1 = [...incrementals].sort((a, b) => + a.name.localeCompare(b.name), + ); + + for (const migration of sorted1) { + const [{ existing }] = await tx` + select exists + ( + select 1 + from ${tx(schema)}.migration + where name = ${migration.name} + ) as existing + `; + + if (existing) { + continue; + } + + await span("space.migrate.incremental", { + attributes: { + "db.schema": schema, + "space.migration": migration.name, + "space.migration_type": "incremental", + "space.target_version": options.targetVersion, + }, + callback: async () => { + const renderedSql = template( + migration.sql, + templateVars(schema, options), + ); + await tx.unsafe(renderedSql); + await tx` + insert into ${tx(schema)}.migration (name, applied_at_version) + values (${migration.name}, ${options.targetVersion})`; + }, + }); + info("Space migration applied", { + "db.schema": schema, + "space.migration": migration.name, + "space.migration_type": "incremental", + "space.target_version": options.targetVersion, + }); + } + + // run idempotent migrations + const sorted2 = [...idempotents].sort((a, b) => a.name.localeCompare(b.name)); + + for (const migration of sorted2) { + await span("space.migrate.idempotent", { + attributes: { + "db.schema": schema, + "space.migration": migration.name, + "space.migration_type": "idempotent", + "space.target_version": options.targetVersion, + }, + callback: async () => { + const renderedSql = template( + migration.sql, + templateVars(schema, options), + ); + await tx.unsafe(renderedSql); + }, + }); + } + + // update version + await tx`update ${tx(schema)}.version set version = ${options.targetVersion}, at = now()`; +} + +async function assertSchemaOwnership(tx: SQL, schema: string): Promise { + const [result] = await tx` + select + n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner + from pg_catalog.pg_namespace n + where n.nspname = ${schema} + `; + + if (!result?.is_owner) { + throw new Error( + `Only the owner of the ${schema} schema can run database migrations`, + ); + } +} + +function template(sql: string, vars: Record): string { + return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { + if (!(key in vars)) { + throw new Error(`Missing template variable: ${key}`); + } + return String(vars[key]); + }); +} diff --git a/packages/space/package.json b/packages/space/package.json new file mode 100644 index 0000000..92f1a9d --- /dev/null +++ b/packages/space/package.json @@ -0,0 +1,9 @@ +{ + "name": "@memory.build/space", + "version": "0.2.5", + "private": true, + "type": "module", + "dependencies": { + "@pydantic/logfire-node": "^0.13.1" + } +} diff --git a/packages/space/slug.ts b/packages/space/slug.ts new file mode 100644 index 0000000..067b323 --- /dev/null +++ b/packages/space/slug.ts @@ -0,0 +1,18 @@ +const SPACE_SCHEMA_RE = /^me_[a-z0-9]{12}$/; +const SLUG_RE = /^[a-z0-9]{12}$/; + +export function isValidSpaceSchema(name: string): boolean { + return SPACE_SCHEMA_RE.test(name); +} + +export function isValidSlug(slug: string): boolean { + return SLUG_RE.test(slug); +} + +export function slugToSchema(slug: string): string { + return `me_${slug}`; +} + +export function schemaToSlug(schema: string): string { + return schema.slice(3); +} diff --git a/packages/space/tsconfig.json b/packages/space/tsconfig.json new file mode 100644 index 0000000..23b1d27 --- /dev/null +++ b/packages/space/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["**/*.ts", "**/*.d.ts"] +} From f32cd9bf39307d71af16e8187b7e768fc97743d8 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 13:58:44 -0500 Subject: [PATCH 002/156] memory search takes effective tree access as arg --- .../migrate/idempotent/000_update.sql | 0 packages/space/migrate/idempotent/002_search.sql | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) rename packages/{space => core}/migrate/idempotent/000_update.sql (100%) diff --git a/packages/space/migrate/idempotent/000_update.sql b/packages/core/migrate/idempotent/000_update.sql similarity index 100% rename from packages/space/migrate/idempotent/000_update.sql rename to packages/core/migrate/idempotent/000_update.sql diff --git a/packages/space/migrate/idempotent/002_search.sql b/packages/space/migrate/idempotent/002_search.sql index 2790f70..cfbf6fe 100644 --- a/packages/space/migrate/idempotent/002_search.sql +++ b/packages/space/migrate/idempotent/002_search.sql @@ -2,7 +2,7 @@ -- search_memory ------------------------------------------------------------------------------- create or replace function {{schema}}.search_memory -( _user_id uuid +( _tree_access jsonb , _bm25 bm25query default null , _vec halfvec({{embedding_dimensions}}) default null , _max_vec_dist float8 default null @@ -170,9 +170,9 @@ begin $sql$ with x as materialized ( - select a.tree_path - from {{schema}}.calc_tree_access($1) a - where a.access >= 1 + select x.tree_path + from jsonb_to_recordset(_tree_access) x(tree_path ltree, access int) + where x.access >= 1 ) select m.id @@ -214,7 +214,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- hybrid_search_memory ------------------------------------------------------------------------------- create or replace function {{schema}}.hybrid_search_memory -( _user_id uuid +( _tree_access jsonb , _bm25 bm25query , _vec halfvec({{embedding_dimensions}}) , _max_vec_dist float8 default null @@ -285,7 +285,7 @@ begin row_number() over (order by m.score desc, m.id) as rank , m.* from {{schema}}.search_memory - ( _user_id => _user_id + ( _tree_access => _tree_access , _bm25 => _bm25 , _ltree => _ltree , _lquery => _lquery @@ -305,7 +305,7 @@ begin row_number() over (order by m.score desc, m.id) as rank , m.* from {{schema}}.search_memory - ( _user_id => _user_id + ( _tree_access => _tree_access , _vec => _vec , _max_vec_dist => _max_vec_dist , _ltree => _ltree From 249f4056abdd1efbc60f79e1903f96a6ca843fa9 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 15:51:06 -0500 Subject: [PATCH 003/156] most tables modeled --- packages/core/index.ts | 1 + .../core/migrate/idempotent/000_update.sql | 31 +++++++--- .../migrate/idempotent/001_tree_access.sql | 0 .../core/migrate/incremental/001_shard.sql | 9 +++ .../core/migrate/incremental/002_space.sql | 13 ++++ .../migrate/incremental/003_principal.sql | 40 +++++++++++++ .../incremental/004_principal_space.sql | 13 ++++ .../migrate/incremental/005_group_member.sql | 17 ++++++ .../migrate/incremental/006_tree_access.sql | 15 +++++ .../core/migrate/incremental/007_api_key.sql | 13 ++++ packages/core/migrate/migrate.ts | 59 +++++++++++++------ packages/core/version.ts | 1 + packages/space/index.ts | 1 + packages/space/migrate/migrate.ts | 30 +++++----- packages/space/version.ts | 1 + 15 files changed, 203 insertions(+), 41 deletions(-) create mode 100644 packages/core/migrate/idempotent/001_tree_access.sql create mode 100644 packages/core/migrate/incremental/001_shard.sql create mode 100644 packages/core/migrate/incremental/002_space.sql create mode 100644 packages/core/migrate/incremental/003_principal.sql create mode 100644 packages/core/migrate/incremental/004_principal_space.sql create mode 100644 packages/core/migrate/incremental/005_group_member.sql create mode 100644 packages/core/migrate/incremental/006_tree_access.sql create mode 100644 packages/core/migrate/incremental/007_api_key.sql create mode 100644 packages/core/version.ts create mode 100644 packages/space/version.ts diff --git a/packages/core/index.ts b/packages/core/index.ts index 8861a32..fc637ec 100644 --- a/packages/core/index.ts +++ b/packages/core/index.ts @@ -1 +1,2 @@ export { type MigrateCoreOptions, migrateCore } from "./migrate/migrate"; +export { CORE_SCHEMA_VERSION } from "./version"; diff --git a/packages/core/migrate/idempotent/000_update.sql b/packages/core/migrate/idempotent/000_update.sql index a102d9e..e42e0c5 100644 --- a/packages/core/migrate/idempotent/000_update.sql +++ b/packages/core/migrate/idempotent/000_update.sql @@ -1,5 +1,5 @@ -- generic trigger function to update updated_at timestamp -create or replace function {{schema}}.update_updated_at() +create or replace function core.update_updated_at() returns trigger as $func$ begin @@ -7,14 +7,29 @@ begin return new; end; $func$ language plpgsql volatile security definer -set search_path to {{schema}}, pg_temp; +set search_path to core, pg_temp; -create or replace trigger memory_before_update_trg -before update on {{schema}}.memory +create or replace trigger space_before_update_trg +before update on core.space for each row -execute function {{schema}}.memory_before_update(); +execute function core.space_before_update(); -create or replace trigger embedding_queue_before_update_trg -before update on {{schema}}.embedding_queue +create or replace trigger principal_before_update_trg +before update on core.principal for each row -execute function {{schema}}.embedding_queue_before_update(); +execute function core.principal_before_update(); + +create or replace trigger principal_space_before_update_trg +before update on core.principal_space +for each row +execute function core.principal_space_before_update(); + +create or replace trigger group_member_before_update_trg +before update on core.group_member +for each row +execute function core.group_member_before_update(); + +create or replace trigger tree_access_before_update_trg +before update on core.tree_access +for each row +execute function core.tree_access_before_update(); diff --git a/packages/core/migrate/idempotent/001_tree_access.sql b/packages/core/migrate/idempotent/001_tree_access.sql new file mode 100644 index 0000000..e69de29 diff --git a/packages/core/migrate/incremental/001_shard.sql b/packages/core/migrate/incremental/001_shard.sql new file mode 100644 index 0000000..138900e --- /dev/null +++ b/packages/core/migrate/incremental/001_shard.sql @@ -0,0 +1,9 @@ +------------------------------------------------------------------------------- +-- shard +------------------------------------------------------------------------------- +create table core.shard +( id int primary key +); + +-- seed default shard +insert into core.shard (id) values (1); diff --git a/packages/core/migrate/incremental/002_space.sql b/packages/core/migrate/incremental/002_space.sql new file mode 100644 index 0000000..e21bf27 --- /dev/null +++ b/packages/core/migrate/incremental/002_space.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- space +------------------------------------------------------------------------------- +create table core.space +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, slug text not null unique check (slug ~ '^[a-z0-9]{12}$') +, name citext not null +, shard_id int not null references {{schema}}.shard +, language text not null default 'english' check (language ~ '^[a-z_]+$') +-- we likely need columns for embedding provider, model, dimensions +, created_at timestamptz not null default now() +, updated_at timestamptz +); diff --git a/packages/core/migrate/incremental/003_principal.sql b/packages/core/migrate/incremental/003_principal.sql new file mode 100644 index 0000000..c3ca2ee --- /dev/null +++ b/packages/core/migrate/incremental/003_principal.sql @@ -0,0 +1,40 @@ +------------------------------------------------------------------------------- +-- principal +------------------------------------------------------------------------------- +create table core.principal +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid generated always as (case when kind = 'user' then id else null end) stored +, group_id uuid generated always as (case when kind = 'group' then id else null end) stored +, agent_id uuid generated always as (case when kind = 'agent' then id else null end) stored +, member_id uuid generated always as (case when kind in ('user', 'agent') then id else null end) stored +, owner_id uuid references core.principal (user_id) on delete cascade -- points to agent's owner +, space_id uuid references core.space (id) on delete cascade +, kind text not null check (kind in ('group', 'user', 'agent')) +, name citext not null check (name::text !~ '/') +, created_at timestamptz not null default now() +, updated_at timestamptz +, check + ( + (kind = 'agent' and owner_id is not null) -- agents are owned by a user + or + (kind != 'agent' and owner_id is null) -- users and groups have no owner + ) +, check + ( + (kind = 'group' and space_id is not null) -- groups belong to a single space + or + (kind != 'group' and space_id is null) -- users and agents are global + ) +); + +create unique index on core.principal (user_id) where user_id is not null; +create unique index on core.principal (group_id) include (space_id) where group_id is not null; +create unique index on core.principal (agent_id) where agent_id is not null; +create unique index on core.principal (member_id) where member_id is not null; + +-- users must have a globally unique name +create unique index on core.principal (name) where user_id is not null; +-- each user's agents must have a unique name (per that user) +create unique index on core.principal (owner_id, name) where agent_id is not null; +-- each space's groups must have a unique name (per that space) +create unique index on core.principal (space_id, name) include (group_id) where group_id is not null; diff --git a/packages/core/migrate/incremental/004_principal_space.sql b/packages/core/migrate/incremental/004_principal_space.sql new file mode 100644 index 0000000..a00b996 --- /dev/null +++ b/packages/core/migrate/incremental/004_principal_space.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- principal_space +------------------------------------------------------------------------------- +create table core.principal_space +( space_id uuid not null references core.space (id) on delete cascade +, principal_id uuid not null references core.principal (id) on delete cascade -- can be users, agents, or groups +, admin bool not null default false +, created_at timestamptz not null default now() +, updated_at timestamptz +, unique (principal_id, space_id) include (admin) +); + +create index on core.principal_space (space_id, principal_id) include (admin); diff --git a/packages/core/migrate/incremental/005_group_member.sql b/packages/core/migrate/incremental/005_group_member.sql new file mode 100644 index 0000000..6c51871 --- /dev/null +++ b/packages/core/migrate/incremental/005_group_member.sql @@ -0,0 +1,17 @@ +------------------------------------------------------------------------------- +-- group_member +------------------------------------------------------------------------------- +create table core.group_member +( space_id uuid not null references core.space (id) on delete cascade +, group_id uuid not null references core.principal (group_id) on delete cascade -- can only be groups +, member_id uuid not null references core.principal (member_id) on delete cascade -- can be users or agents, but not groups +, admin bool not null default false +, created_at timestamptz not null default now() +, updated_at timestamptz +, unique (member_id, space_id, group_id) include (admin) +); + +-- index for listing members of a group +create index on core.group_member (group_id, member_id) include (admin); +-- index for listing groups in a space +create index on core.group_member (space_id, group_id); diff --git a/packages/core/migrate/incremental/006_tree_access.sql b/packages/core/migrate/incremental/006_tree_access.sql new file mode 100644 index 0000000..4552bc9 --- /dev/null +++ b/packages/core/migrate/incremental/006_tree_access.sql @@ -0,0 +1,15 @@ +------------------------------------------------------------------------------- +-- tree_access +------------------------------------------------------------------------------- +create table core.tree_access +( space_id uuid not null references core.space (id) on delete cascade +, principal_id uuid not null references core.principal (id) on delete cascade -- can be users, agents, or groups +, tree_path ltree not null +, access int not null check (access in (1, 2, 3)) -- 1 = read, 2 = write, 3 = owner +, created_at timestamptz not null default now() +, updated_at timestamptz +, unique (principal_id, space_id, tree_path) include (access) +); + +-- list access per space or per space and principal +create index on core.tree_access (space_id, principal_id); diff --git a/packages/core/migrate/incremental/007_api_key.sql b/packages/core/migrate/incremental/007_api_key.sql new file mode 100644 index 0000000..85b7250 --- /dev/null +++ b/packages/core/migrate/incremental/007_api_key.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- api_key +------------------------------------------------------------------------------- +create table core.api_key +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, member_id uuid not null references core.principal (member_id) on delete cascade -- may be users or agents, not groups +, lookup_id text unique not null check (lookup_id ~ '^[A-Za-z0-9_-]{16}$') +, secret text not null -- hashed secret +, name text not null +, created_at timestamptz not null default now() +, expires_at timestamptz +, unique (member_id, name) +); diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index 954eaa0..379ec46 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -1,24 +1,50 @@ import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; import { SQL, semver } from "bun"; +import { CORE_SCHEMA_VERSION } from "../version"; import provisionSql from "./incremental/000_provision.sql" with { type: "text", }; +import incremental001 from "./incremental/001_shard.sql" with { type: "text" }; +import incremental002 from "./incremental/002_space.sql" with { type: "text" }; +import incremental003 from "./incremental/003_principal.sql" with { + type: "text", +}; +import incremental004 from "./incremental/004_principal_space.sql" with { + type: "text", +}; +import incremental005 from "./incremental/005_group_member.sql" with { + type: "text", +}; +import incremental006 from "./incremental/006_tree_access.sql" with { + type: "text", +}; +import incremental007 from "./incremental/007_api_key.sql" with { type: "text" }; interface Incremental { name: string; sql: string; } -const incrementals: Incremental[] = []; +const incrementals: Incremental[] = [ + { name: "001_shard", sql: incremental001 }, + { name: "002_space", sql: incremental002 }, + { name: "003_principal", sql: incremental003 }, + { name: "004_principal_space", sql: incremental004 }, + { name: "005_group_member", sql: incremental005 }, + { name: "006_tree_access", sql: incremental006 }, + { name: "007_api_key", sql: incremental007 }, +]; + +import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; interface Idempotent { name: string; sql: string; } -const idempotents: Idempotent[] = []; +const idempotents: Idempotent[] = [{ name: "000_update", sql: idempotent000 }]; const CORE_SCHEMA = "core"; const REQUIRED_EXTENSIONS = [ @@ -29,7 +55,6 @@ const REQUIRED_EXTENSIONS = [ ] as const; export interface MigrateCoreOptions { - targetVersion: string; statementTimeout?: string; lockTimeout?: string; transactionTimeout?: string; @@ -37,7 +62,7 @@ export interface MigrateCoreOptions { } interface NormalizedMigrateCoreOptions { - targetVersion: string; + schemaVersion: string; statementTimeout: string; lockTimeout: string; transactionTimeout: string; @@ -46,7 +71,7 @@ interface NormalizedMigrateCoreOptions { export async function migrateCore( sql: SQL, - options: MigrateCoreOptions, + options: MigrateCoreOptions = {}, ): Promise { const opts = normalizeMigrateCoreOptions(options); const attributes = migrateAttributes(opts); @@ -55,8 +80,8 @@ export async function migrateCore( attributes, callback: async () => { try { - if (!semver.satisfies(opts.targetVersion, "*")) { - throw new Error(`Invalid target version: "${opts.targetVersion}"`); + if (!semver.satisfies(opts.schemaVersion, "*")) { + throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); } const [key1, key2] = advisoryLockKey("memory-core:schema:core"); @@ -111,7 +136,7 @@ function migrateAttributes( ): Record { return { "db.schema": CORE_SCHEMA, - "core.target_version": options.targetVersion, + "core.schema_version": options.schemaVersion, "core.required_extensions": REQUIRED_EXTENSIONS.map( (extension) => `${extension.name}@>=${extension.minVersion}`, ), @@ -127,7 +152,7 @@ function normalizeMigrateCoreOptions( options: MigrateCoreOptions, ): NormalizedMigrateCoreOptions { return { - targetVersion: options.targetVersion, + schemaVersion: CORE_SCHEMA_VERSION, statementTimeout: options.statementTimeout ?? "20s", lockTimeout: options.lockTimeout ?? "5s", transactionTimeout: options.transactionTimeout ?? "1min", @@ -258,10 +283,10 @@ async function runMigrations( const [{ version: dbVersion }] = await tx` select version from core.version `; - const cmp = semver.order(options.targetVersion, dbVersion); + const cmp = semver.order(options.schemaVersion, dbVersion); if (cmp < 0) { throw new Error( - `Target version (${options.targetVersion}) is older than database version (${dbVersion}). ` + + `Schema version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + "Please upgrade the server.", ); } @@ -269,7 +294,7 @@ async function runMigrations( info("Core migration skipped, version current", { "db.schema": CORE_SCHEMA, "core.version": dbVersion, - "core.target_version": options.targetVersion, + "core.schema_version": options.schemaVersion, }); return; } @@ -297,20 +322,20 @@ async function runMigrations( "db.schema": CORE_SCHEMA, "core.migration": migration.name, "core.migration_type": "incremental", - "core.target_version": options.targetVersion, + "core.schema_version": options.schemaVersion, }, callback: async () => { await tx.unsafe(migration.sql); await tx` insert into core.migration (name, applied_at_version) - values (${migration.name}, ${options.targetVersion})`; + values (${migration.name}, ${options.schemaVersion})`; }, }); info("Core migration applied", { "db.schema": CORE_SCHEMA, "core.migration": migration.name, "core.migration_type": "incremental", - "core.target_version": options.targetVersion, + "core.schema_version": options.schemaVersion, }); } @@ -322,13 +347,13 @@ async function runMigrations( "db.schema": CORE_SCHEMA, "core.migration": migration.name, "core.migration_type": "idempotent", - "core.target_version": options.targetVersion, + "core.schema_version": options.schemaVersion, }, callback: () => tx.unsafe(migration.sql), }); } - await tx`update core.version set version = ${options.targetVersion}, at = now()`; + await tx`update core.version set version = ${options.schemaVersion}, at = now()`; } async function assertSchemaOwnership(tx: SQL): Promise { diff --git a/packages/core/version.ts b/packages/core/version.ts new file mode 100644 index 0000000..9fd61d7 --- /dev/null +++ b/packages/core/version.ts @@ -0,0 +1 @@ +export const CORE_SCHEMA_VERSION = "0.0.1"; diff --git a/packages/space/index.ts b/packages/space/index.ts index 3a7ed38..2ca99e8 100644 --- a/packages/space/index.ts +++ b/packages/space/index.ts @@ -6,3 +6,4 @@ export { schemaToSlug, slugToSchema, } from "./slug"; +export { SPACE_SCHEMA_VERSION } from "./version"; diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts index 70fa7fd..efe614d 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/space/migrate/migrate.ts @@ -3,6 +3,7 @@ import { info, reportError, span } from "@pydantic/logfire-node"; import type { SQL } from "bun"; import { semver } from "bun"; import { isValidSlug, slugToSchema } from "../slug"; +import { SPACE_SCHEMA_VERSION } from "../version"; import provisionSql from "./incremental/000_provision.sql" with { type: "text", @@ -22,7 +23,6 @@ const incrementals: Incremental[] = [ { name: "002_embedding_queue", sql: incremental002 }, ]; -import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; import idempotent001 from "./idempotent/001_memory.sql" with { type: "text" }; import idempotent002 from "./idempotent/002_search.sql" with { type: "text" }; import idempotent003 from "./idempotent/003_embedding_queue.sql" with { @@ -35,7 +35,6 @@ interface Idempotent { } const idempotents: Idempotent[] = [ - { name: "000_update", sql: idempotent000 }, { name: "001_memory", sql: idempotent001 }, { name: "002_search", sql: idempotent002 }, { name: "003_embedding_queue", sql: idempotent003 }, @@ -43,7 +42,6 @@ const idempotents: Idempotent[] = [ export interface MigrateSpaceOptions { slug: string; - targetVersion: string; shardId?: number; embeddingDimensions?: number; bm25TextConfig?: string; @@ -59,7 +57,7 @@ export interface MigrateSpaceOptions { interface NormalizedMigrateSpaceOptions { slug: string; - targetVersion: string; + schemaVersion: string; shardId?: number; embeddingDimensions: number; bm25TextConfig: string; @@ -89,8 +87,8 @@ export async function migrateSpace( `Invalid space slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, ); } - if (!semver.satisfies(opts.targetVersion, "*")) { - throw new Error(`Invalid target version: "${opts.targetVersion}"`); + if (!semver.satisfies(opts.schemaVersion, "*")) { + throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); } const schema = slugToSchema(opts.slug); const schemaAttributes = { ...attributes, "db.schema": schema }; @@ -145,7 +143,7 @@ function migrateAttributes( ): Record { return { "space.slug": options.slug, - "space.target_version": options.targetVersion, + "space.schema_version": options.schemaVersion, "db.shard": options.shardId, "db.statement_timeout": options.statementTimeout, "db.lock_timeout": options.lockTimeout, @@ -160,7 +158,7 @@ function normalizeMigrateSpaceOptions( ): NormalizedMigrateSpaceOptions { return { slug: options.slug, - targetVersion: options.targetVersion, + schemaVersion: SPACE_SCHEMA_VERSION, shardId: options.shardId, embeddingDimensions: options.embeddingDimensions ?? 1536, bm25TextConfig: options.bm25TextConfig ?? "english", @@ -253,11 +251,11 @@ async function runMigrations( const [{ version: dbVersion }] = await tx` select version from ${tx(schema)}.version `; - const cmp = semver.order(options.targetVersion, dbVersion); + const cmp = semver.order(options.schemaVersion, dbVersion); // abort if target is older than the database if (cmp < 0) { throw new Error( - `Target version (${options.targetVersion}) is older than database version (${dbVersion}). ` + + `Schema version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + "Please upgrade the server.", ); } @@ -266,7 +264,7 @@ async function runMigrations( info("Space migration skipped, version current", { "db.schema": schema, "space.version": dbVersion, - "space.target_version": options.targetVersion, + "space.schema_version": options.schemaVersion, }); return; } @@ -295,7 +293,7 @@ async function runMigrations( "db.schema": schema, "space.migration": migration.name, "space.migration_type": "incremental", - "space.target_version": options.targetVersion, + "space.schema_version": options.schemaVersion, }, callback: async () => { const renderedSql = template( @@ -305,14 +303,14 @@ async function runMigrations( await tx.unsafe(renderedSql); await tx` insert into ${tx(schema)}.migration (name, applied_at_version) - values (${migration.name}, ${options.targetVersion})`; + values (${migration.name}, ${options.schemaVersion})`; }, }); info("Space migration applied", { "db.schema": schema, "space.migration": migration.name, "space.migration_type": "incremental", - "space.target_version": options.targetVersion, + "space.schema_version": options.schemaVersion, }); } @@ -325,7 +323,7 @@ async function runMigrations( "db.schema": schema, "space.migration": migration.name, "space.migration_type": "idempotent", - "space.target_version": options.targetVersion, + "space.schema_version": options.schemaVersion, }, callback: async () => { const renderedSql = template( @@ -338,7 +336,7 @@ async function runMigrations( } // update version - await tx`update ${tx(schema)}.version set version = ${options.targetVersion}, at = now()`; + await tx`update ${tx(schema)}.version set version = ${options.schemaVersion}, at = now()`; } async function assertSchemaOwnership(tx: SQL, schema: string): Promise { diff --git a/packages/space/version.ts b/packages/space/version.ts new file mode 100644 index 0000000..1b92647 --- /dev/null +++ b/packages/space/version.ts @@ -0,0 +1 @@ +export const SPACE_SCHEMA_VERSION = "0.0.1"; From fbdfee15ffba0a614d1402f2785f978523e5fdb7 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 16:38:23 -0500 Subject: [PATCH 004/156] runtime calculation of tree access --- .../core/migrate/idempotent/000_update.sql | 10 +- .../migrate/idempotent/001_tree_access.sql | 133 ++++++++++++++++++ packages/core/migrate/migrate.ts | 8 +- 3 files changed, 145 insertions(+), 6 deletions(-) diff --git a/packages/core/migrate/idempotent/000_update.sql b/packages/core/migrate/idempotent/000_update.sql index e42e0c5..53a4ce6 100644 --- a/packages/core/migrate/idempotent/000_update.sql +++ b/packages/core/migrate/idempotent/000_update.sql @@ -12,24 +12,24 @@ set search_path to core, pg_temp; create or replace trigger space_before_update_trg before update on core.space for each row -execute function core.space_before_update(); +execute function core.update_updated_at(); create or replace trigger principal_before_update_trg before update on core.principal for each row -execute function core.principal_before_update(); +execute function core.update_updated_at(); create or replace trigger principal_space_before_update_trg before update on core.principal_space for each row -execute function core.principal_space_before_update(); +execute function core.update_updated_at(); create or replace trigger group_member_before_update_trg before update on core.group_member for each row -execute function core.group_member_before_update(); +execute function core.update_updated_at(); create or replace trigger tree_access_before_update_trg before update on core.tree_access for each row -execute function core.tree_access_before_update(); +execute function core.update_updated_at(); diff --git a/packages/core/migrate/idempotent/001_tree_access.sql b/packages/core/migrate/idempotent/001_tree_access.sql index e69de29..92718aa 100644 --- a/packages/core/migrate/idempotent/001_tree_access.sql +++ b/packages/core/migrate/idempotent/001_tree_access.sql @@ -0,0 +1,133 @@ +------------------------------------------------------------------------------- +-- member_tree_access +------------------------------------------------------------------------------- +create or replace function core.member_tree_access +( _member_id uuid +, _space_id uuid +) +returns table +( tree_path ltree +, access int +) +as $func$ + -- member's grants via groups + select + ta.tree_path + , ta.access + from core.principal m + inner join core.principal_space psu on (m.id = psu.principal_id and psu.space_id = _space_id) + inner join core.group_member gm on (m.member_id = gm.member_id and gm.space_id = _space_id) + inner join core.principal g on (gm.group_id = g.group_id and g.space_id = _space_id) + inner join core.principal_space psg on (g.id = psg.principal_id and psg.space_id = _space_id) + inner join core.tree_access ta on (g.id = ta.principal_id and ta.space_id = _space_id) + where m.member_id = _member_id +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- user_tree_access +------------------------------------------------------------------------------- +create or replace function core.user_tree_access +( _user_id uuid +, _space_id uuid +) +returns table +( tree_path ltree +, access int +) +as $func$ + -- user's direct grants + select + ta.tree_path + , ta.access + from core.principal u + inner join core.principal_space psu on (u.id = psu.principal_id and psu.space_id = _space_id) + inner join core.tree_access ta on (u.id = ta.principal_id and ta.space_id = _space_id) + where u.user_id = _user_id + union + -- user's access via groups + select + x.tree_path + , x.access + from core.member_tree_access(_user_id, _space_id) x +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- agent_tree_access +------------------------------------------------------------------------------- +create or replace function core.agent_tree_access +( _agent_id uuid +, _space_id uuid +) +returns table +( tree_path ltree +, access int +) +as $func$ + with agent_access as materialized + ( + -- agent's direct grants + select + ta.tree_path + , ta.access + from core.principal a + inner join core.principal_space ps on (a.id = ps.principal_id and ps.space_id = _space_id) + inner join core.tree_access ta on (a.id = ta.principal_id and ta.space_id = _space_id) + where a.agent_id = _agent_id + union + -- agent's access via groups + select + x.tree_path + , x.access + from core.member_tree_access(_agent_id, _space_id) x + ) + , owner_access as materialized + ( + -- get the access for the user that owns the agent + select + x.tree_path + , x.access + from + ( + select p.owner_id + from core.principal p + where p.agent_id = _agent_id + ) a + cross join lateral core.user_tree_access(a.owner_id, _space_id) x + ) + select + x.tree_path + , max(x.access) + from + ( + -- take the agent's access when it is covered by the owner's access + select + aa.tree_path + , aa.access + from agent_access aa + where exists + ( + -- the owner must have access that is the same or greater than the agent's + select 1 + from owner_access oa + where oa.tree_path @> aa.tree_path + and oa.access >= aa.access + ) + union + -- when the agent has more access than the owner, take the owner's access + select + oa.tree_path + , oa.access + from owner_access oa + where exists + ( + select 1 + from agent_access aa + where aa.tree_path @> oa.tree_path + and aa.access >= oa.access + ) + ) x + group by x.tree_path +$func$ language sql stable security invoker +; diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index 379ec46..26eed4e 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -38,13 +38,19 @@ const incrementals: Incremental[] = [ ]; import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; +import idempotent001 from "./idempotent/001_tree_access.sql" with { + type: "text", +}; interface Idempotent { name: string; sql: string; } -const idempotents: Idempotent[] = [{ name: "000_update", sql: idempotent000 }]; +const idempotents: Idempotent[] = [ + { name: "000_update", sql: idempotent000 }, + { name: "001_tree_access", sql: idempotent001 }, +]; const CORE_SCHEMA = "core"; const REQUIRED_EXTENSIONS = [ From b1b041d4b9ce42096a5c5bf51a5a1ccd4ddde999 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 17:36:09 -0500 Subject: [PATCH 005/156] fix database migrations and diagnostics --- bun.lock | 18 +- package.json | 1 + .../core/migrate/incremental/002_space.sql | 2 +- .../migrate/incremental/003_principal.sql | 13 +- packages/core/migrate/migrate.ts | 180 ++++++++++++++++-- .../space/migrate/idempotent/001_memory.sql | 80 +++++--- .../space/migrate/idempotent/002_search.sql | 17 +- packages/space/migrate/migrate.ts | 167 ++++++++++++++-- scripts/migrate-db.ts | 143 ++++++++++++++ scripts/package.json | 2 + 10 files changed, 547 insertions(+), 76 deletions(-) create mode 100644 scripts/migrate-db.ts diff --git a/bun.lock b/bun.lock index 0302f5f..6f341f7 100644 --- a/bun.lock +++ b/bun.lock @@ -14,14 +14,14 @@ }, "packages/accounts": { "name": "@memory.build/accounts", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@pydantic/logfire-node": "^0.13.1", }, }, "packages/cli": { "name": "@memory.build/cli", - "version": "0.2.5", + "version": "0.2.6", "bin": { "me": "./index.ts", }, @@ -38,7 +38,7 @@ }, "packages/client": { "name": "@memory.build/client", - "version": "0.2.5", + "version": "0.2.6", "dependencies": { "@memory.build/protocol": "workspace:*", }, @@ -90,7 +90,7 @@ }, "packages/embedding": { "name": "@memory.build/embedding", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@ai-sdk/openai": "^3.0.0", "@pydantic/logfire-node": "^0.13.1", @@ -99,14 +99,14 @@ }, "packages/engine": { "name": "@memory.build/engine", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@pydantic/logfire-node": "^0.13.1", }, }, "packages/protocol": { "name": "@memory.build/protocol", - "version": "0.2.5", + "version": "0.2.6", "dependencies": { "zod": "^4.0.0", }, @@ -119,7 +119,7 @@ }, "packages/server": { "name": "memory-engine-server", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@memory.build/accounts": "workspace:*", "@memory.build/embedding": "workspace:*", @@ -167,7 +167,7 @@ }, "packages/worker": { "name": "@memory.build/worker", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@memory.build/embedding": "workspace:*", "@memory.build/engine": "workspace:*", @@ -177,7 +177,9 @@ "scripts": { "name": "scripts", "dependencies": { + "@memory.build/core": "workspace:*", "@memory.build/embedding": "workspace:*", + "@memory.build/space": "workspace:*", "yaml": "^2.7.0", }, }, diff --git a/package.json b/package.json index 29e8f2f..400f49a 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "install:local": "./bun scripts/install-local.ts", "lint": "biome check", "me": "./bun run packages/cli/index.ts", + "migrate:db": "./bun scripts/migrate-db.ts", "pg": "./bun run pg:build && docker run -d --name me-postgres -e POSTGRES_HOST_AUTH_METHOD=trust -p 127.0.0.1:5432:5432 me-postgres", "pg:build": "docker build -t me-postgres -f docker/Dockerfile.postgres docker/", "pg:rm": "docker rm -f me-postgres", diff --git a/packages/core/migrate/incremental/002_space.sql b/packages/core/migrate/incremental/002_space.sql index e21bf27..fa8cf22 100644 --- a/packages/core/migrate/incremental/002_space.sql +++ b/packages/core/migrate/incremental/002_space.sql @@ -5,7 +5,7 @@ create table core.space ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) , slug text not null unique check (slug ~ '^[a-z0-9]{12}$') , name citext not null -, shard_id int not null references {{schema}}.shard +, shard_id int not null references core.shard (id) , language text not null default 'english' check (language ~ '^[a-z_]+$') -- we likely need columns for embedding provider, model, dimensions , created_at timestamptz not null default now() diff --git a/packages/core/migrate/incremental/003_principal.sql b/packages/core/migrate/incremental/003_principal.sql index c3ca2ee..e5ae87a 100644 --- a/packages/core/migrate/incremental/003_principal.sql +++ b/packages/core/migrate/incremental/003_principal.sql @@ -3,10 +3,10 @@ ------------------------------------------------------------------------------- create table core.principal ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) -, user_id uuid generated always as (case when kind = 'user' then id else null end) stored -, group_id uuid generated always as (case when kind = 'group' then id else null end) stored -, agent_id uuid generated always as (case when kind = 'agent' then id else null end) stored -, member_id uuid generated always as (case when kind in ('user', 'agent') then id else null end) stored +, user_id uuid unique nulls distinct generated always as (case when kind = 'user' then id else null end) stored +, group_id uuid unique nulls distinct generated always as (case when kind = 'group' then id else null end) stored +, agent_id uuid unique nulls distinct generated always as (case when kind = 'agent' then id else null end) stored +, member_id uuid unique nulls distinct generated always as (case when kind in ('user', 'agent') then id else null end) stored , owner_id uuid references core.principal (user_id) on delete cascade -- points to agent's owner , space_id uuid references core.space (id) on delete cascade , kind text not null check (kind in ('group', 'user', 'agent')) @@ -27,11 +27,6 @@ create table core.principal ) ); -create unique index on core.principal (user_id) where user_id is not null; -create unique index on core.principal (group_id) include (space_id) where group_id is not null; -create unique index on core.principal (agent_id) where agent_id is not null; -create unique index on core.principal (member_id) where member_id is not null; - -- users must have a globally unique name create unique index on core.principal (name) where user_id is not null; -- each user's agents must have a unique name (per that user) diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index 26eed4e..7a34b61 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -20,21 +20,44 @@ import incremental005 from "./incremental/005_group_member.sql" with { import incremental006 from "./incremental/006_tree_access.sql" with { type: "text", }; -import incremental007 from "./incremental/007_api_key.sql" with { type: "text" }; +import incremental007 from "./incremental/007_api_key.sql" with { + type: "text", +}; interface Incremental { name: string; + file: string; sql: string; } const incrementals: Incremental[] = [ - { name: "001_shard", sql: incremental001 }, - { name: "002_space", sql: incremental002 }, - { name: "003_principal", sql: incremental003 }, - { name: "004_principal_space", sql: incremental004 }, - { name: "005_group_member", sql: incremental005 }, - { name: "006_tree_access", sql: incremental006 }, - { name: "007_api_key", sql: incremental007 }, + { name: "001_shard", file: "incremental/001_shard.sql", sql: incremental001 }, + { name: "002_space", file: "incremental/002_space.sql", sql: incremental002 }, + { + name: "003_principal", + file: "incremental/003_principal.sql", + sql: incremental003, + }, + { + name: "004_principal_space", + file: "incremental/004_principal_space.sql", + sql: incremental004, + }, + { + name: "005_group_member", + file: "incremental/005_group_member.sql", + sql: incremental005, + }, + { + name: "006_tree_access", + file: "incremental/006_tree_access.sql", + sql: incremental006, + }, + { + name: "007_api_key", + file: "incremental/007_api_key.sql", + sql: incremental007, + }, ]; import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; @@ -44,12 +67,17 @@ import idempotent001 from "./idempotent/001_tree_access.sql" with { interface Idempotent { name: string; + file: string; sql: string; } const idempotents: Idempotent[] = [ - { name: "000_update", sql: idempotent000 }, - { name: "001_tree_access", sql: idempotent001 }, + { name: "000_update", file: "idempotent/000_update.sql", sql: idempotent000 }, + { + name: "001_tree_access", + file: "idempotent/001_tree_access.sql", + sql: idempotent001, + }, ]; const CORE_SCHEMA = "core"; @@ -61,6 +89,7 @@ const REQUIRED_EXTENSIONS = [ ] as const; export interface MigrateCoreOptions { + logSqlFiles?: boolean; statementTimeout?: string; lockTimeout?: string; transactionTimeout?: string; @@ -68,6 +97,7 @@ export interface MigrateCoreOptions { } interface NormalizedMigrateCoreOptions { + logSqlFiles: boolean; schemaVersion: string; statementTimeout: string; lockTimeout: string; @@ -118,8 +148,12 @@ export async function migrateCore( if (!(await doesCoreExist(tx))) { await span("core.migrate.provision", { - attributes, - callback: () => provisionCore(tx), + attributes: { + ...attributes, + "core.migration_file": "incremental/000_provision.sql", + "core.migration_type": "provision", + }, + callback: () => provisionCore(tx, opts), }); info("Core schema provisioned", attributes); } @@ -158,6 +192,7 @@ function normalizeMigrateCoreOptions( options: MigrateCoreOptions, ): NormalizedMigrateCoreOptions { return { + logSqlFiles: options.logSqlFiles ?? false, schemaVersion: CORE_SCHEMA_VERSION, statementTimeout: options.statementTimeout ?? "20s", lockTimeout: options.lockTimeout ?? "5s", @@ -212,8 +247,17 @@ async function doesCoreExist(tx: SQL): Promise { return coreExists; } -async function provisionCore(tx: SQL): Promise { - await tx.unsafe(provisionSql); +async function provisionCore( + tx: SQL, + options: NormalizedMigrateCoreOptions, +): Promise { + await executeSqlFile( + tx, + options, + "provision", + "incremental/000_provision.sql", + provisionSql, + ); } async function ensurePostgresVersion(tx: SQL): Promise { @@ -327,11 +371,18 @@ async function runMigrations( attributes: { "db.schema": CORE_SCHEMA, "core.migration": migration.name, + "core.migration_file": migration.file, "core.migration_type": "incremental", "core.schema_version": options.schemaVersion, }, callback: async () => { - await tx.unsafe(migration.sql); + await executeSqlFile( + tx, + options, + "incremental", + migration.file, + migration.sql, + ); await tx` insert into core.migration (name, applied_at_version) values (${migration.name}, ${options.schemaVersion})`; @@ -340,6 +391,7 @@ async function runMigrations( info("Core migration applied", { "db.schema": CORE_SCHEMA, "core.migration": migration.name, + "core.migration_file": migration.file, "core.migration_type": "incremental", "core.schema_version": options.schemaVersion, }); @@ -352,16 +404,112 @@ async function runMigrations( attributes: { "db.schema": CORE_SCHEMA, "core.migration": migration.name, + "core.migration_file": migration.file, "core.migration_type": "idempotent", "core.schema_version": options.schemaVersion, }, - callback: () => tx.unsafe(migration.sql), + callback: () => + executeSqlFile( + tx, + options, + "idempotent", + migration.file, + migration.sql, + ), }); } await tx`update core.version set version = ${options.schemaVersion}, at = now()`; } +async function executeSqlFile( + tx: SQL, + options: NormalizedMigrateCoreOptions, + type: string, + file: string, + sqlText: string, +): Promise { + logSqlFile(options, type, file); + try { + await tx.unsafe(sqlText); + } catch (error) { + logSqlExecutionError(options, type, file, sqlText, error); + throw error; + } +} + +function logSqlFile( + options: NormalizedMigrateCoreOptions, + type: string, + file: string, +): void { + if (!options.logSqlFiles) return; + console.error(`[migrate:db] core ${type} packages/core/migrate/${file}`); +} + +function logSqlExecutionError( + options: NormalizedMigrateCoreOptions, + type: string, + file: string, + sqlText: string, + error: unknown, +): void { + if (!options.logSqlFiles) return; + console.error( + `[migrate:db] failed core ${type} packages/core/migrate/${file}`, + ); + logPostgresSqlLocation(sqlText, error); +} + +function logPostgresSqlLocation(sqlText: string, error: unknown): void { + if (!(error instanceof SQL.PostgresError)) return; + const position = Number(error.position); + if (!Number.isSafeInteger(position) || position < 1) return; + + const location = sqlLocation(sqlText, position); + if (!location) return; + console.error( + `[migrate:db] sql position ${position} -> line ${location.line}, column ${location.column}`, + ); + console.error(sqlContext(sqlText, location.line, location.column)); +} + +function sqlLocation( + sqlText: string, + position: number, +): { line: number; column: number } | undefined { + if (position > sqlText.length + 1) return undefined; + let line = 1; + let column = 1; + for (let i = 0; i < position - 1; i++) { + if (sqlText.charCodeAt(i) === 10) { + line++; + column = 1; + } else { + column++; + } + } + return { line, column }; +} + +function sqlContext(sqlText: string, line: number, column: number): string { + const lines = sqlText.split("\n"); + const start = Math.max(1, line - 2); + const end = Math.min(lines.length, line + 2); + const width = String(end).length; + const output = ["[migrate:db] sql context:"]; + + for (let n = start; n <= end; n++) { + const marker = n === line ? ">" : " "; + output.push(`${marker} ${String(n).padStart(width)} | ${lines[n - 1]}`); + if (n === line) { + output.push(` ${" ".repeat(width)} | ${" ".repeat(column - 1)}^`); + } + } + + return output.join("\n"); +} + async function assertSchemaOwnership(tx: SQL): Promise { const [result] = await tx` select diff --git a/packages/space/migrate/idempotent/001_memory.sql b/packages/space/migrate/idempotent/001_memory.sql index 6de72b3..c4048d8 100644 --- a/packages/space/migrate/idempotent/001_memory.sql +++ b/packages/space/migrate/idempotent/001_memory.sql @@ -27,11 +27,47 @@ before update on {{schema}}.memory for each row execute function {{schema}}.memory_before_update(); +------------------------------------------------------------------------------- +-- tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.tree_access(_tree_access jsonb) +returns table +( tree_path ltree +, access int +) +as $func$ + select + x.tree_path + , x.access + from jsonb_to_recordset(_tree_access) x(tree_path ltree, access int) +$func$ language sql immutable strict security invoker +; + +------------------------------------------------------------------------------- +-- has_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.has_tree_access +( _tree_access jsonb +, _tree_path ltree +, _access int +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.tree_access(_tree_access) x + where x.tree_path @> _tree_path + and x.access >= _access + ) +$func$ language sql immutable strict security invoker +; + ------------------------------------------------------------------------------- -- get memory ------------------------------------------------------------------------------- create or replace function {{schema}}.get_memory -( _user_id uuid +( _tree_access jsonb , _id uuid default null ) returns table @@ -56,7 +92,7 @@ as $func$ , m.embedding is not null from {{schema}}.memory m where m.id = _id - and {{schema}}.has_tree_access(_user_id, m.tree, 1) + and {{schema}}.has_tree_access(_tree_access, m.tree, 1) $func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -65,7 +101,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- create memory ------------------------------------------------------------------------------- create or replace function {{schema}}.create_memory -( _user_id uuid +( _tree_access jsonb , _tree ltree , _content text , _id uuid default null @@ -75,7 +111,7 @@ create or replace function {{schema}}.create_memory returns uuid as $func$ begin - if not {{schema}}.has_tree_access(_user_id, _tree, 2) then + if not {{schema}}.has_tree_access(_tree_access, _tree, 2) then raise exception 'insufficient tree access' using errcode = 'insufficient_privilege'; end if; @@ -106,7 +142,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- patch memory ------------------------------------------------------------------------------- create or replace function {{schema}}.patch_memory -( _user_id uuid +( _tree_access jsonb , _id uuid , _patch jsonb ) @@ -150,7 +186,7 @@ begin with a as materialized ( select a.tree_path, a.access - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a ) select exists @@ -199,7 +235,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- move tree ------------------------------------------------------------------------------- create or replace function {{schema}}.move_tree -( _user_id uuid +( _tree_access jsonb , _src ltree , _dst ltree , _dry_run bool default false @@ -216,7 +252,7 @@ begin with a as materialized ( select a.tree_path, a.access - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a ) select exists @@ -277,7 +313,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- copy tree ------------------------------------------------------------------------------- create or replace function {{schema}}.copy_tree -( _user_id uuid +( _tree_access jsonb , _src ltree , _dst ltree , _dry_run bool default false @@ -294,7 +330,7 @@ begin with a as materialized ( select a.tree_path, a.access - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a ) select exists @@ -367,7 +403,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- delete memory ------------------------------------------------------------------------------- create or replace function {{schema}}.delete_memory -( _user_id uuid +( _tree_access jsonb , _id uuid ) returns bool @@ -385,7 +421,7 @@ begin return false; end if; - if not {{schema}}.has_tree_access(_user_id, _tree, 2) then + if not {{schema}}.has_tree_access(_tree_access, _tree, 2) then raise exception 'insufficient tree access' using errcode = 'insufficient_privilege'; end if; @@ -403,7 +439,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- delete tree ------------------------------------------------------------------------------- create or replace function {{schema}}.delete_tree -( _user_id uuid +( _tree_access jsonb , _tree ltree , _dry_run bool default false ) @@ -417,7 +453,7 @@ begin select exists ( select 1 - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a where a.tree_path @> _tree and a.access >= 2 ) @@ -456,7 +492,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- count tree ------------------------------------------------------------------------------- create or replace function {{schema}}.count_tree -( _user_id uuid +( _tree_access jsonb , _tree ltree , _access int4 ) @@ -465,7 +501,7 @@ as $func$ with x as materialized ( select a.tree_path - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a where a.access >= _access ) select count(*) @@ -485,7 +521,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- count tree ------------------------------------------------------------------------------- create or replace function {{schema}}.count_tree -( _user_id uuid +( _tree_access jsonb , _query lquery , _access int4 ) @@ -494,7 +530,7 @@ as $func$ with x as materialized ( select a.tree_path - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a where a.access >= _access ) select count(*) @@ -514,7 +550,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- count tree ------------------------------------------------------------------------------- create or replace function {{schema}}.count_tree -( _user_id uuid +( _tree_access jsonb , _query ltxtquery , _access int4 ) @@ -523,7 +559,7 @@ as $func$ with x as materialized ( select a.tree_path - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a where a.access >= _access ) select count(*) @@ -543,7 +579,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- list tree ------------------------------------------------------------------------------- create or replace function {{schema}}.list_tree -( _user_id uuid +( _tree_access jsonb , _query lquery ) returns table @@ -554,7 +590,7 @@ as $func$ with a as materialized ( select a.tree_path - from {{schema}}.calc_tree_access(_user_id) a + from {{schema}}.tree_access(_tree_access) a where a.access >= 1 ) , m as diff --git a/packages/space/migrate/idempotent/002_search.sql b/packages/space/migrate/idempotent/002_search.sql index cfbf6fe..6da0335 100644 --- a/packages/space/migrate/idempotent/002_search.sql +++ b/packages/space/migrate/idempotent/002_search.sql @@ -78,7 +78,7 @@ begin end if; else _score = $sql$, -1 as score$sql$; - _order_by = $sql$order by m.id; + _order_by = $sql$order by m.id$sql$; end case; -- ltree @@ -171,7 +171,7 @@ begin with x as materialized ( select x.tree_path - from jsonb_to_recordset(_tree_access) x(tree_path ltree, access int) + from jsonb_to_recordset($1) x(tree_path ltree, access int) where x.access >= 1 ) select @@ -197,14 +197,17 @@ begin $sql$ , _score , coalesce - (( - select string_agg(x, E'\n ') - from unnest(_filters) x - ), '') + ( + ( + select string_agg(x, E'\n ') + from unnest(_filters) x + ) + , '' + ) , _order_by ); - return query execute _sql using _user_id, _limit; + return query execute _sql using _tree_access, _limit; end; $func$ language plpgsql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts index efe614d..75a62b4 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/space/migrate/migrate.ts @@ -1,7 +1,6 @@ import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import { semver } from "bun"; +import { SQL, semver } from "bun"; import { isValidSlug, slugToSchema } from "../slug"; import { SPACE_SCHEMA_VERSION } from "../version"; @@ -15,12 +14,21 @@ import incremental002 from "./incremental/002_embedding_queue.sql" with { interface Incremental { name: string; + file: string; sql: string; } const incrementals: Incremental[] = [ - { name: "001_memory", sql: incremental001 }, - { name: "002_embedding_queue", sql: incremental002 }, + { + name: "001_memory", + file: "incremental/001_memory.sql", + sql: incremental001, + }, + { + name: "002_embedding_queue", + file: "incremental/002_embedding_queue.sql", + sql: incremental002, + }, ]; import idempotent001 from "./idempotent/001_memory.sql" with { type: "text" }; @@ -31,17 +39,23 @@ import idempotent003 from "./idempotent/003_embedding_queue.sql" with { interface Idempotent { name: string; + file: string; sql: string; } const idempotents: Idempotent[] = [ - { name: "001_memory", sql: idempotent001 }, - { name: "002_search", sql: idempotent002 }, - { name: "003_embedding_queue", sql: idempotent003 }, + { name: "001_memory", file: "idempotent/001_memory.sql", sql: idempotent001 }, + { name: "002_search", file: "idempotent/002_search.sql", sql: idempotent002 }, + { + name: "003_embedding_queue", + file: "idempotent/003_embedding_queue.sql", + sql: idempotent003, + }, ]; export interface MigrateSpaceOptions { slug: string; + logSqlFiles?: boolean; shardId?: number; embeddingDimensions?: number; bm25TextConfig?: string; @@ -57,6 +71,7 @@ export interface MigrateSpaceOptions { interface NormalizedMigrateSpaceOptions { slug: string; + logSqlFiles: boolean; schemaVersion: string; shardId?: number; embeddingDimensions: number; @@ -119,8 +134,12 @@ export async function migrateSpace( if (!(await doesSpaceExist(tx, schema))) { await span("space.migrate.provision", { - attributes: schemaAttributes, - callback: () => provisionSpace(tx, schema), + attributes: { + ...schemaAttributes, + "space.migration_file": "incremental/000_provision.sql", + "space.migration_type": "provision", + }, + callback: () => provisionSpace(tx, schema, opts), }); info("Space schema provisioned", schemaAttributes); } @@ -158,6 +177,7 @@ function normalizeMigrateSpaceOptions( ): NormalizedMigrateSpaceOptions { return { slug: options.slug, + logSqlFiles: options.logSqlFiles ?? false, schemaVersion: SPACE_SCHEMA_VERSION, shardId: options.shardId, embeddingDimensions: options.embeddingDimensions ?? 1536, @@ -235,8 +255,19 @@ async function doesSpaceExist(tx: SQL, schema: string): Promise { return spaceExists; } -async function provisionSpace(tx: SQL, schema: string): Promise { - await tx.unsafe(template(provisionSql, { schema })); +async function provisionSpace( + tx: SQL, + schema: string, + options: NormalizedMigrateSpaceOptions, +): Promise { + await executeSqlFile( + tx, + options, + schema, + "provision", + "incremental/000_provision.sql", + template(provisionSql, { schema }), + ); } async function runMigrations( @@ -292,6 +323,7 @@ async function runMigrations( attributes: { "db.schema": schema, "space.migration": migration.name, + "space.migration_file": migration.file, "space.migration_type": "incremental", "space.schema_version": options.schemaVersion, }, @@ -300,7 +332,14 @@ async function runMigrations( migration.sql, templateVars(schema, options), ); - await tx.unsafe(renderedSql); + await executeSqlFile( + tx, + options, + schema, + "incremental", + migration.file, + renderedSql, + ); await tx` insert into ${tx(schema)}.migration (name, applied_at_version) values (${migration.name}, ${options.schemaVersion})`; @@ -309,6 +348,7 @@ async function runMigrations( info("Space migration applied", { "db.schema": schema, "space.migration": migration.name, + "space.migration_file": migration.file, "space.migration_type": "incremental", "space.schema_version": options.schemaVersion, }); @@ -322,6 +362,7 @@ async function runMigrations( attributes: { "db.schema": schema, "space.migration": migration.name, + "space.migration_file": migration.file, "space.migration_type": "idempotent", "space.schema_version": options.schemaVersion, }, @@ -330,7 +371,14 @@ async function runMigrations( migration.sql, templateVars(schema, options), ); - await tx.unsafe(renderedSql); + await executeSqlFile( + tx, + options, + schema, + "idempotent", + migration.file, + renderedSql, + ); }, }); } @@ -339,6 +387,99 @@ async function runMigrations( await tx`update ${tx(schema)}.version set version = ${options.schemaVersion}, at = now()`; } +async function executeSqlFile( + tx: SQL, + options: NormalizedMigrateSpaceOptions, + schema: string, + type: string, + file: string, + sqlText: string, +): Promise { + logSqlFile(options, schema, type, file); + try { + await tx.unsafe(sqlText); + } catch (error) { + logSqlExecutionError(options, schema, type, file, sqlText, error); + throw error; + } +} + +function logSqlFile( + options: NormalizedMigrateSpaceOptions, + schema: string, + type: string, + file: string, +): void { + if (!options.logSqlFiles) return; + console.error( + `[migrate:db] space ${schema} ${type} packages/space/migrate/${file}`, + ); +} + +function logSqlExecutionError( + options: NormalizedMigrateSpaceOptions, + schema: string, + type: string, + file: string, + sqlText: string, + error: unknown, +): void { + if (!options.logSqlFiles) return; + console.error( + `[migrate:db] failed space ${schema} ${type} packages/space/migrate/${file}`, + ); + logPostgresSqlLocation(sqlText, error); +} + +function logPostgresSqlLocation(sqlText: string, error: unknown): void { + if (!(error instanceof SQL.PostgresError)) return; + const position = Number(error.position); + if (!Number.isSafeInteger(position) || position < 1) return; + + const location = sqlLocation(sqlText, position); + if (!location) return; + console.error( + `[migrate:db] sql position ${position} -> line ${location.line}, column ${location.column}`, + ); + console.error(sqlContext(sqlText, location.line, location.column)); +} + +function sqlLocation( + sqlText: string, + position: number, +): { line: number; column: number } | undefined { + if (position > sqlText.length + 1) return undefined; + let line = 1; + let column = 1; + for (let i = 0; i < position - 1; i++) { + if (sqlText.charCodeAt(i) === 10) { + line++; + column = 1; + } else { + column++; + } + } + return { line, column }; +} + +function sqlContext(sqlText: string, line: number, column: number): string { + const lines = sqlText.split("\n"); + const start = Math.max(1, line - 2); + const end = Math.min(lines.length, line + 2); + const width = String(end).length; + const output = ["[migrate:db] sql context:"]; + + for (let n = start; n <= end; n++) { + const marker = n === line ? ">" : " "; + output.push(`${marker} ${String(n).padStart(width)} | ${lines[n - 1]}`); + if (n === line) { + output.push(` ${" ".repeat(width)} | ${" ".repeat(column - 1)}^`); + } + } + + return output.join("\n"); +} + async function assertSchemaOwnership(tx: SQL, schema: string): Promise { const [result] = await tx` select diff --git a/scripts/migrate-db.ts b/scripts/migrate-db.ts new file mode 100644 index 0000000..471deca --- /dev/null +++ b/scripts/migrate-db.ts @@ -0,0 +1,143 @@ +#!/usr/bin/env bun +import { CORE_SCHEMA_VERSION, migrateCore } from "@memory.build/core"; +import { + bootstrapSpaceDatabase, + migrateSpace, + SPACE_SCHEMA_VERSION, + slugToSchema, +} from "@memory.build/space"; +import { SQL } from "bun"; + +const DEFAULT_DATABASE_URL = "postgresql://postgres@127.0.0.1:5432/postgres"; +const DEFAULT_SPACE_SLUG = "dev000000001"; + +type Mode = "all" | "core" | "space-db" | "space"; + +function usage(): string { + return `Usage: ./bun run migrate:db [all|core|space-db|space] + +Environment: + DATABASE_URL Postgres connection string. Falls back to ENGINE_DATABASE_URL, then ${DEFAULT_DATABASE_URL} + SPACE_SLUG Space slug to migrate. Defaults to ${DEFAULT_SPACE_SLUG} + +Modes: + all Migrate core, prepare database for spaces, and migrate the dev space. Default. + core Migrate only the core schema. + space-db Prepare only the physical database for spaces. + space Prepare the database for spaces and migrate one space. +`; +} + +function parseMode(arg: string | undefined): Mode { + if (!arg) return "all"; + if (arg === "--help" || arg === "-h") { + console.log(usage()); + process.exit(0); + } + if ( + arg === "all" || + arg === "core" || + arg === "space-db" || + arg === "space" + ) { + return arg; + } + console.error(`Invalid migration mode: ${arg}`); + console.error(usage()); + process.exit(1); +} + +function databaseUrl(): string { + return ( + process.env.DATABASE_URL ?? + process.env.ENGINE_DATABASE_URL ?? + DEFAULT_DATABASE_URL + ); +} + +async function main(): Promise { + const mode = parseMode(process.argv[2]); + const url = databaseUrl(); + const spaceSlug = process.env.SPACE_SLUG ?? DEFAULT_SPACE_SLUG; + const sql = new SQL(url); + + console.log(`Database: ${url}`); + console.log(`Mode: ${mode}`); + console.log(`Space slug: ${spaceSlug}`); + console.log(`Core schema version: ${CORE_SCHEMA_VERSION}`); + console.log(`Space schema version: ${SPACE_SCHEMA_VERSION}`); + console.log(""); + + try { + if (mode === "all" || mode === "core") { + await migrateCore(sql, { logSqlFiles: true }); + console.log("Migrated core."); + } + + if (mode === "all" || mode === "space-db" || mode === "space") { + await bootstrapSpaceDatabase(sql); + console.log("Prepared database for spaces."); + } + + if (mode === "all" || mode === "space") { + await migrateSpace(sql, { slug: spaceSlug, logSqlFiles: true }); + console.log(`Migrated space ${slugToSchema(spaceSlug)}.`); + } + } finally { + await sql.close(); + } +} + +main().catch((error) => { + console.error(""); + console.error( + "Migration failed:", + error instanceof Error ? error.message : error, + ); + printErrorDetails(error); + process.exit(1); +}); + +function printErrorDetails(error: unknown): void { + if (!error || typeof error !== "object") return; + + const details = error as Record; + const keys = [ + "name", + "errno", + "code", + "severity", + "detail", + "hint", + "position", + "internalPosition", + "internalQuery", + "where", + "schema", + "table", + "column", + "dataType", + "constraint", + "file", + "line", + "routine", + ]; + const seen = new Set(keys); + const extraKeys = [ + ...Object.getOwnPropertyNames(error), + ...Object.keys(details), + ].filter((key) => !seen.has(key) && key !== "message" && key !== "stack"); + + const entries = [...keys, ...extraKeys] + .map((key) => [key, details[key]] as const) + .filter( + ([, value]) => value !== undefined && value !== null && value !== "", + ); + + if (entries.length === 0) return; + + console.error("Postgres error details:"); + for (const [key, value] of entries) { + console.error(` ${key}: ${String(value)}`); + } +} diff --git a/scripts/package.json b/scripts/package.json index 8bd79d2..8919499 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -3,7 +3,9 @@ "private": true, "type": "module", "dependencies": { + "@memory.build/core": "workspace:*", "@memory.build/embedding": "workspace:*", + "@memory.build/space": "workspace:*", "yaml": "^2.7.0" } } From 1c11069144b8efcf85e2b56fdcbc6ff7e70944e4 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 18:21:38 -0500 Subject: [PATCH 006/156] split core idempotent migrations --- .../idempotent/001_principal_space.sql | 41 +++++++++++++++++++ .../migrate/idempotent/002_group_member.sql | 26 ++++++++++++ ...01_tree_access.sql => 003_tree_access.sql} | 9 +--- packages/core/migrate/migrate.ts | 22 ++++++++-- 4 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 packages/core/migrate/idempotent/001_principal_space.sql create mode 100644 packages/core/migrate/idempotent/002_group_member.sql rename packages/core/migrate/idempotent/{001_tree_access.sql => 003_tree_access.sql} (86%) diff --git a/packages/core/migrate/idempotent/001_principal_space.sql b/packages/core/migrate/idempotent/001_principal_space.sql new file mode 100644 index 0000000..b096a4f --- /dev/null +++ b/packages/core/migrate/idempotent/001_principal_space.sql @@ -0,0 +1,41 @@ +------------------------------------------------------------------------------- +-- is_principal_in_space +------------------------------------------------------------------------------- +create or replace function core.is_principal_in_space +( _principal_id uuid +, _space_id uuid +) +returns bool +as $func$ + select exists + ( + select 1 + from core.principal_space ps + where ps.principal_id = _principal_id + and ps.space_id = _space_id + ) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- is_principal_space_admin +------------------------------------------------------------------------------- +create or replace function core.is_principal_space_admin +( _principal_id uuid +, _space_id uuid +) +returns bool +as $func$ + select coalesce + ( + ( + select ps.admin and (not p.kind = 'agent') -- agents cannot be space admins + from core.principal_space ps + inner join core.principal p on (ps.principal_id = p.id) + where ps.principal_id = _principal_id + and ps.space_id = _space_id + ) + , false + ) +$func$ language sql stable security invoker +; diff --git a/packages/core/migrate/idempotent/002_group_member.sql b/packages/core/migrate/idempotent/002_group_member.sql new file mode 100644 index 0000000..19a884e --- /dev/null +++ b/packages/core/migrate/idempotent/002_group_member.sql @@ -0,0 +1,26 @@ + +------------------------------------------------------------------------------- +-- member_groups +------------------------------------------------------------------------------- +create or replace function core.member_groups +( _member_id uuid +, _space_id uuid +) +returns table +( group_id uuid +, admin bool +) +as $func$ + select + gm.group_id + , gm.admin and (not m.kind = 'agent') -- agent's cannot be group admins + from core.principal m -- the member + -- assert the member belongs to the space + inner join core.principal_space psm on (m.id = psm.principal_id and psm.space_id = _space_id) + -- find the groups the member belongs to in the space + inner join core.group_member gm on (m.member_id = gm.member_id and gm.space_id = _space_id) + -- assert the group belongs to the space + inner join core.principal_space psg on (gm.group_id = psg.principal_id and psg.space_id = _space_id) + where m.member_id = _member_id -- the member +$func$ language sql stable security invoker +; diff --git a/packages/core/migrate/idempotent/001_tree_access.sql b/packages/core/migrate/idempotent/003_tree_access.sql similarity index 86% rename from packages/core/migrate/idempotent/001_tree_access.sql rename to packages/core/migrate/idempotent/003_tree_access.sql index 92718aa..ad804a5 100644 --- a/packages/core/migrate/idempotent/001_tree_access.sql +++ b/packages/core/migrate/idempotent/003_tree_access.sql @@ -14,13 +14,8 @@ as $func$ select ta.tree_path , ta.access - from core.principal m - inner join core.principal_space psu on (m.id = psu.principal_id and psu.space_id = _space_id) - inner join core.group_member gm on (m.member_id = gm.member_id and gm.space_id = _space_id) - inner join core.principal g on (gm.group_id = g.group_id and g.space_id = _space_id) - inner join core.principal_space psg on (g.id = psg.principal_id and psg.space_id = _space_id) - inner join core.tree_access ta on (g.id = ta.principal_id and ta.space_id = _space_id) - where m.member_id = _member_id + from core.member_groups(_member_id, _space_id) mg + inner join core.tree_access ta on (mg.group_id = ta.principal_id and ta.space_id = _space_id) $func$ language sql stable security invoker ; diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index 7a34b61..8da85bd 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -61,7 +61,13 @@ const incrementals: Incremental[] = [ ]; import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; -import idempotent001 from "./idempotent/001_tree_access.sql" with { +import idempotent001 from "./idempotent/001_principal_space.sql" with { + type: "text", +}; +import idempotent002 from "./idempotent/002_group_member.sql" with { + type: "text", +}; +import idempotent003 from "./idempotent/003_tree_access.sql" with { type: "text", }; @@ -74,10 +80,20 @@ interface Idempotent { const idempotents: Idempotent[] = [ { name: "000_update", file: "idempotent/000_update.sql", sql: idempotent000 }, { - name: "001_tree_access", - file: "idempotent/001_tree_access.sql", + name: "001_principal_space", + file: "idempotent/001_principal_space.sql", sql: idempotent001, }, + { + name: "002_group_member", + file: "idempotent/002_group_member.sql", + sql: idempotent002, + }, + { + name: "003_tree_access", + file: "idempotent/003_tree_access.sql", + sql: idempotent003, + }, ]; const CORE_SCHEMA = "core"; From fa2597e988d59ec0e4500b98b4de087ebcc36b7e Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 18:22:56 -0500 Subject: [PATCH 007/156] docs: clarify principal agent names --- packages/core/migrate/incremental/003_principal.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/migrate/incremental/003_principal.sql b/packages/core/migrate/incremental/003_principal.sql index e5ae87a..db0d97d 100644 --- a/packages/core/migrate/incremental/003_principal.sql +++ b/packages/core/migrate/incremental/003_principal.sql @@ -10,7 +10,7 @@ create table core.principal , owner_id uuid references core.principal (user_id) on delete cascade -- points to agent's owner , space_id uuid references core.space (id) on delete cascade , kind text not null check (kind in ('group', 'user', 'agent')) -, name citext not null check (name::text !~ '/') +, name citext not null check (name::text !~ '/') -- agent names are displayed as / , created_at timestamptz not null default now() , updated_at timestamptz , check From 4c8376cee95be5e743e17a524a73a4897deac99d Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 28 May 2026 18:31:27 -0500 Subject: [PATCH 008/156] shorten principal kind values --- .../migrate/idempotent/001_principal_space.sql | 2 +- .../migrate/idempotent/002_group_member.sql | 2 +- .../core/migrate/incremental/003_principal.sql | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/core/migrate/idempotent/001_principal_space.sql b/packages/core/migrate/idempotent/001_principal_space.sql index b096a4f..b221c4b 100644 --- a/packages/core/migrate/idempotent/001_principal_space.sql +++ b/packages/core/migrate/idempotent/001_principal_space.sql @@ -29,7 +29,7 @@ as $func$ select coalesce ( ( - select ps.admin and (not p.kind = 'agent') -- agents cannot be space admins + select ps.admin and (not p.kind = 'a') -- agents cannot be space admins from core.principal_space ps inner join core.principal p on (ps.principal_id = p.id) where ps.principal_id = _principal_id diff --git a/packages/core/migrate/idempotent/002_group_member.sql b/packages/core/migrate/idempotent/002_group_member.sql index 19a884e..f7e32c7 100644 --- a/packages/core/migrate/idempotent/002_group_member.sql +++ b/packages/core/migrate/idempotent/002_group_member.sql @@ -13,7 +13,7 @@ returns table as $func$ select gm.group_id - , gm.admin and (not m.kind = 'agent') -- agent's cannot be group admins + , gm.admin and (not m.kind = 'a') -- agent's cannot be group admins from core.principal m -- the member -- assert the member belongs to the space inner join core.principal_space psm on (m.id = psm.principal_id and psm.space_id = _space_id) diff --git a/packages/core/migrate/incremental/003_principal.sql b/packages/core/migrate/incremental/003_principal.sql index db0d97d..51eb2e9 100644 --- a/packages/core/migrate/incremental/003_principal.sql +++ b/packages/core/migrate/incremental/003_principal.sql @@ -3,27 +3,27 @@ ------------------------------------------------------------------------------- create table core.principal ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) -, user_id uuid unique nulls distinct generated always as (case when kind = 'user' then id else null end) stored -, group_id uuid unique nulls distinct generated always as (case when kind = 'group' then id else null end) stored -, agent_id uuid unique nulls distinct generated always as (case when kind = 'agent' then id else null end) stored -, member_id uuid unique nulls distinct generated always as (case when kind in ('user', 'agent') then id else null end) stored +, user_id uuid unique nulls distinct generated always as (case when kind = 'u' then id else null end) stored +, group_id uuid unique nulls distinct generated always as (case when kind = 'g' then id else null end) stored +, agent_id uuid unique nulls distinct generated always as (case when kind = 'a' then id else null end) stored +, member_id uuid unique nulls distinct generated always as (case when kind in ('u', 'a') then id else null end) stored , owner_id uuid references core.principal (user_id) on delete cascade -- points to agent's owner , space_id uuid references core.space (id) on delete cascade -, kind text not null check (kind in ('group', 'user', 'agent')) +, kind text not null check (kind in ('g', 'u', 'a')) -- group, user, agent , name citext not null check (name::text !~ '/') -- agent names are displayed as / , created_at timestamptz not null default now() , updated_at timestamptz , check ( - (kind = 'agent' and owner_id is not null) -- agents are owned by a user + (kind = 'a' and owner_id is not null) -- agents are owned by a user or - (kind != 'agent' and owner_id is null) -- users and groups have no owner + (kind != 'a' and owner_id is null) -- users and groups have no owner ) , check ( - (kind = 'group' and space_id is not null) -- groups belong to a single space + (kind = 'g' and space_id is not null) -- groups belong to a single space or - (kind != 'group' and space_id is null) -- users and agents are global + (kind != 'g' and space_id is null) -- users and agents are global ) ); From 32ad49e96c917ece0adc1a77410bbf8ac129d20a Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 08:11:43 -0500 Subject: [PATCH 009/156] chore: make provision scripts a separate thing from incrementals --- packages/core/migrate/migrate.ts | 6 +++--- .../{incremental/000_provision.sql => provision.sql} | 0 packages/core/migrate/sql.d.ts | 4 ++++ packages/space/migrate/migrate.ts | 6 +++--- .../{incremental/000_provision.sql => provision.sql} | 0 packages/space/migrate/sql.d.ts | 4 ++++ 6 files changed, 14 insertions(+), 6 deletions(-) rename packages/core/migrate/{incremental/000_provision.sql => provision.sql} (100%) create mode 100644 packages/core/migrate/sql.d.ts rename packages/space/migrate/{incremental/000_provision.sql => provision.sql} (100%) create mode 100644 packages/space/migrate/sql.d.ts diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index 8da85bd..0939c91 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -3,7 +3,7 @@ import { info, reportError, span } from "@pydantic/logfire-node"; import { SQL, semver } from "bun"; import { CORE_SCHEMA_VERSION } from "../version"; -import provisionSql from "./incremental/000_provision.sql" with { +import provisionSql from "./provision.sql" with { type: "text", }; import incremental001 from "./incremental/001_shard.sql" with { type: "text" }; @@ -166,7 +166,7 @@ export async function migrateCore( await span("core.migrate.provision", { attributes: { ...attributes, - "core.migration_file": "incremental/000_provision.sql", + "core.migration_file": "provision.sql", "core.migration_type": "provision", }, callback: () => provisionCore(tx, opts), @@ -271,7 +271,7 @@ async function provisionCore( tx, options, "provision", - "incremental/000_provision.sql", + "provision.sql", provisionSql, ); } diff --git a/packages/core/migrate/incremental/000_provision.sql b/packages/core/migrate/provision.sql similarity index 100% rename from packages/core/migrate/incremental/000_provision.sql rename to packages/core/migrate/provision.sql diff --git a/packages/core/migrate/sql.d.ts b/packages/core/migrate/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/core/migrate/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts index 75a62b4..88ede83 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/space/migrate/migrate.ts @@ -4,7 +4,7 @@ import { SQL, semver } from "bun"; import { isValidSlug, slugToSchema } from "../slug"; import { SPACE_SCHEMA_VERSION } from "../version"; -import provisionSql from "./incremental/000_provision.sql" with { +import provisionSql from "./provision.sql" with { type: "text", }; import incremental001 from "./incremental/001_memory.sql" with { type: "text" }; @@ -136,7 +136,7 @@ export async function migrateSpace( await span("space.migrate.provision", { attributes: { ...schemaAttributes, - "space.migration_file": "incremental/000_provision.sql", + "space.migration_file": "provision.sql", "space.migration_type": "provision", }, callback: () => provisionSpace(tx, schema, opts), @@ -265,7 +265,7 @@ async function provisionSpace( options, schema, "provision", - "incremental/000_provision.sql", + "provision.sql", template(provisionSql, { schema }), ); } diff --git a/packages/space/migrate/incremental/000_provision.sql b/packages/space/migrate/provision.sql similarity index 100% rename from packages/space/migrate/incremental/000_provision.sql rename to packages/space/migrate/provision.sql diff --git a/packages/space/migrate/sql.d.ts b/packages/space/migrate/sql.d.ts new file mode 100644 index 0000000..89b092e --- /dev/null +++ b/packages/space/migrate/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const content: string; + export default content; +} From f8d811d86e504218ca0ca3c1e9bafd8c8efa7150 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 08:27:31 -0500 Subject: [PATCH 010/156] chore: simplify extension creation --- packages/core/migrate/migrate.ts | 26 +++----------------------- packages/space/migrate/bootstrap.ts | 16 ++-------------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index 0939c91..d4a5c0c 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -2,10 +2,6 @@ import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; import { SQL, semver } from "bun"; import { CORE_SCHEMA_VERSION } from "../version"; - -import provisionSql from "./provision.sql" with { - type: "text", -}; import incremental001 from "./incremental/001_shard.sql" with { type: "text" }; import incremental002 from "./incremental/002_space.sql" with { type: "text" }; import incremental003 from "./incremental/003_principal.sql" with { @@ -23,6 +19,7 @@ import incremental006 from "./incremental/006_tree_access.sql" with { import incremental007 from "./incremental/007_api_key.sql" with { type: "text", }; +import provisionSql from "./provision.sql" with { type: "text" }; interface Incremental { name: string; @@ -267,13 +264,7 @@ async function provisionCore( tx: SQL, options: NormalizedMigrateCoreOptions, ): Promise { - await executeSqlFile( - tx, - options, - "provision", - "provision.sql", - provisionSql, - ); + await executeSqlFile(tx, options, "provision", "provision.sql", provisionSql); } async function ensurePostgresVersion(tx: SQL): Promise { @@ -326,18 +317,7 @@ async function ensureExtension( ); } - try { - await tx`create extension if not exists ${tx(name)} with schema public`; - } catch (error: unknown) { - if ( - error instanceof SQL.PostgresError && - error.errno === "23505" && - error.constraint === "pg_extension_name_index" - ) { - return; - } - throw error; - } + await tx`create extension if not exists ${tx(name)} with schema public`; } async function runMigrations( diff --git a/packages/space/migrate/bootstrap.ts b/packages/space/migrate/bootstrap.ts index cc73a12..07d67ec 100644 --- a/packages/space/migrate/bootstrap.ts +++ b/packages/space/migrate/bootstrap.ts @@ -1,5 +1,5 @@ import { info, reportError, span } from "@pydantic/logfire-node"; -import { SQL, semver } from "bun"; +import { type SQL, semver } from "bun"; const REQUIRED_EXTENSIONS = [ { name: "citext", minVersion: "1.6" }, @@ -147,17 +147,5 @@ async function ensureExtension( ); } - try { - await tx`create extension if not exists ${tx(name)} with schema public`; - } catch (error: unknown) { - // Ignore duplicate extension errors (race condition in concurrent calls) - if ( - error instanceof SQL.PostgresError && - error.errno === "23505" && - error.constraint === "pg_extension_name_index" - ) { - return; - } - throw error; - } + await tx`create extension if not exists ${tx(name)} with schema public`; } From 477a29e3f20e3df4865ebba70919289e80b038d3 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 08:39:57 -0500 Subject: [PATCH 011/156] fix check script web asset generation --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 400f49a..7897c2a 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "./bun run --filter '@memory.build/cli' build", "build:all": "./bun scripts/build-all.ts", - "check": "./bun i --silent && ./bun run typecheck && ./bun run lint --write && ./bun run test --only-failures", + "check": "./bun i --silent && ./bun scripts/bundle-web-assets.ts && ./bun run typecheck && ./bun run lint --write && ./bun run test --only-failures", "clean": "rm -rf packages/cli/dist dist", "docs": "./bun --filter @memory.build/docs-site dev", "docs:build": "./bun --filter @memory.build/docs-site build", From 346121f129d7e0847e57655a5fbd311efe15481b Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 08:40:56 -0500 Subject: [PATCH 012/156] chore: format space migration imports --- packages/space/migrate/migrate.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts index 88ede83..c931cc2 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/space/migrate/migrate.ts @@ -3,14 +3,11 @@ import { info, reportError, span } from "@pydantic/logfire-node"; import { SQL, semver } from "bun"; import { isValidSlug, slugToSchema } from "../slug"; import { SPACE_SCHEMA_VERSION } from "../version"; - -import provisionSql from "./provision.sql" with { - type: "text", -}; import incremental001 from "./incremental/001_memory.sql" with { type: "text" }; import incremental002 from "./incremental/002_embedding_queue.sql" with { type: "text", }; +import provisionSql from "./provision.sql" with { type: "text" }; interface Incremental { name: string; From d027c758d513a79b7cfcf75661494cd5bcb5c469 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 08:45:53 -0500 Subject: [PATCH 013/156] run migrations when version is current --- packages/core/migrate/migrate.ts | 6 ++++-- packages/space/migrate/migrate.ts | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index d4a5c0c..56ed5d0 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -332,10 +332,11 @@ async function runMigrations( const cmp = semver.order(options.schemaVersion, dbVersion); if (cmp < 0) { throw new Error( - `Schema version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the server.", + `Application version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the application.", ); } + /* run migrations regardless if (cmp === 0) { info("Core migration skipped, version current", { "db.schema": CORE_SCHEMA, @@ -344,6 +345,7 @@ async function runMigrations( }); return; } + */ const sorted1 = [...incrementals].sort((a, b) => a.name.localeCompare(b.name), diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts index c931cc2..b5154ba 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/space/migrate/migrate.ts @@ -283,10 +283,11 @@ async function runMigrations( // abort if target is older than the database if (cmp < 0) { throw new Error( - `Schema version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the server.", + `Application version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the application.", ); } + /* run migrations regardless if (cmp === 0) { // version matches. no need to run migrations info("Space migration skipped, version current", { @@ -296,6 +297,7 @@ async function runMigrations( }); return; } + */ // run incremental migrations const sorted1 = [...incrementals].sort((a, b) => From 02e38b30ee1cf4278905e658c0a29b7c9c8f7a97 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 09:10:52 -0500 Subject: [PATCH 014/156] remove core shard migration --- .../core/migrate/incremental/001_shard.sql | 9 ---- .../{002_space.sql => 001_space.sql} | 1 - .../{003_principal.sql => 002_principal.sql} | 0 ...ipal_space.sql => 003_principal_space.sql} | 0 ..._group_member.sql => 004_group_member.sql} | 0 ...06_tree_access.sql => 005_tree_access.sql} | 0 .../{007_api_key.sql => 006_api_key.sql} | 0 packages/core/migrate/migrate.ts | 42 +++++++++---------- 8 files changed, 20 insertions(+), 32 deletions(-) delete mode 100644 packages/core/migrate/incremental/001_shard.sql rename packages/core/migrate/incremental/{002_space.sql => 001_space.sql} (91%) rename packages/core/migrate/incremental/{003_principal.sql => 002_principal.sql} (100%) rename packages/core/migrate/incremental/{004_principal_space.sql => 003_principal_space.sql} (100%) rename packages/core/migrate/incremental/{005_group_member.sql => 004_group_member.sql} (100%) rename packages/core/migrate/incremental/{006_tree_access.sql => 005_tree_access.sql} (100%) rename packages/core/migrate/incremental/{007_api_key.sql => 006_api_key.sql} (100%) diff --git a/packages/core/migrate/incremental/001_shard.sql b/packages/core/migrate/incremental/001_shard.sql deleted file mode 100644 index 138900e..0000000 --- a/packages/core/migrate/incremental/001_shard.sql +++ /dev/null @@ -1,9 +0,0 @@ -------------------------------------------------------------------------------- --- shard -------------------------------------------------------------------------------- -create table core.shard -( id int primary key -); - --- seed default shard -insert into core.shard (id) values (1); diff --git a/packages/core/migrate/incremental/002_space.sql b/packages/core/migrate/incremental/001_space.sql similarity index 91% rename from packages/core/migrate/incremental/002_space.sql rename to packages/core/migrate/incremental/001_space.sql index fa8cf22..931d2f7 100644 --- a/packages/core/migrate/incremental/002_space.sql +++ b/packages/core/migrate/incremental/001_space.sql @@ -5,7 +5,6 @@ create table core.space ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) , slug text not null unique check (slug ~ '^[a-z0-9]{12}$') , name citext not null -, shard_id int not null references core.shard (id) , language text not null default 'english' check (language ~ '^[a-z_]+$') -- we likely need columns for embedding provider, model, dimensions , created_at timestamptz not null default now() diff --git a/packages/core/migrate/incremental/003_principal.sql b/packages/core/migrate/incremental/002_principal.sql similarity index 100% rename from packages/core/migrate/incremental/003_principal.sql rename to packages/core/migrate/incremental/002_principal.sql diff --git a/packages/core/migrate/incremental/004_principal_space.sql b/packages/core/migrate/incremental/003_principal_space.sql similarity index 100% rename from packages/core/migrate/incremental/004_principal_space.sql rename to packages/core/migrate/incremental/003_principal_space.sql diff --git a/packages/core/migrate/incremental/005_group_member.sql b/packages/core/migrate/incremental/004_group_member.sql similarity index 100% rename from packages/core/migrate/incremental/005_group_member.sql rename to packages/core/migrate/incremental/004_group_member.sql diff --git a/packages/core/migrate/incremental/006_tree_access.sql b/packages/core/migrate/incremental/005_tree_access.sql similarity index 100% rename from packages/core/migrate/incremental/006_tree_access.sql rename to packages/core/migrate/incremental/005_tree_access.sql diff --git a/packages/core/migrate/incremental/007_api_key.sql b/packages/core/migrate/incremental/006_api_key.sql similarity index 100% rename from packages/core/migrate/incremental/007_api_key.sql rename to packages/core/migrate/incremental/006_api_key.sql diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index 56ed5d0..c9e5fe0 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -2,21 +2,20 @@ import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; import { SQL, semver } from "bun"; import { CORE_SCHEMA_VERSION } from "../version"; -import incremental001 from "./incremental/001_shard.sql" with { type: "text" }; -import incremental002 from "./incremental/002_space.sql" with { type: "text" }; -import incremental003 from "./incremental/003_principal.sql" with { +import incremental001 from "./incremental/001_space.sql" with { type: "text" }; +import incremental002 from "./incremental/002_principal.sql" with { type: "text", }; -import incremental004 from "./incremental/004_principal_space.sql" with { +import incremental003 from "./incremental/003_principal_space.sql" with { type: "text", }; -import incremental005 from "./incremental/005_group_member.sql" with { +import incremental004 from "./incremental/004_group_member.sql" with { type: "text", }; -import incremental006 from "./incremental/006_tree_access.sql" with { +import incremental005 from "./incremental/005_tree_access.sql" with { type: "text", }; -import incremental007 from "./incremental/007_api_key.sql" with { +import incremental006 from "./incremental/006_api_key.sql" with { type: "text", }; import provisionSql from "./provision.sql" with { type: "text" }; @@ -28,33 +27,32 @@ interface Incremental { } const incrementals: Incremental[] = [ - { name: "001_shard", file: "incremental/001_shard.sql", sql: incremental001 }, - { name: "002_space", file: "incremental/002_space.sql", sql: incremental002 }, + { name: "001_space", file: "incremental/001_space.sql", sql: incremental001 }, { - name: "003_principal", - file: "incremental/003_principal.sql", + name: "002_principal", + file: "incremental/002_principal.sql", + sql: incremental002, + }, + { + name: "003_principal_space", + file: "incremental/003_principal_space.sql", sql: incremental003, }, { - name: "004_principal_space", - file: "incremental/004_principal_space.sql", + name: "004_group_member", + file: "incremental/004_group_member.sql", sql: incremental004, }, { - name: "005_group_member", - file: "incremental/005_group_member.sql", + name: "005_tree_access", + file: "incremental/005_tree_access.sql", sql: incremental005, }, { - name: "006_tree_access", - file: "incremental/006_tree_access.sql", + name: "006_api_key", + file: "incremental/006_api_key.sql", sql: incremental006, }, - { - name: "007_api_key", - file: "incremental/007_api_key.sql", - sql: incremental007, - }, ]; import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; From 4f95e24aff1e7ba4500ed4690ced2c189efc692b Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 09:21:34 -0500 Subject: [PATCH 015/156] tune group member indexes --- packages/core/migrate/incremental/004_group_member.sql | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/core/migrate/incremental/004_group_member.sql b/packages/core/migrate/incremental/004_group_member.sql index 6c51871..4821cf6 100644 --- a/packages/core/migrate/incremental/004_group_member.sql +++ b/packages/core/migrate/incremental/004_group_member.sql @@ -8,10 +8,8 @@ create table core.group_member , admin bool not null default false , created_at timestamptz not null default now() , updated_at timestamptz -, unique (member_id, space_id, group_id) include (admin) +, unique (space_id, member_id, group_id) include (admin) ); --- index for listing members of a group -create index on core.group_member (group_id, member_id) include (admin); --- index for listing groups in a space -create index on core.group_member (space_id, group_id); +-- index for listing groups in a space and/or members of a group +create index on core.group_member (space_id, group_id, member_id) include (admin); From 05fdeaaabaf2da9f9ef1aab060ade0f9398b926d Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 09:24:41 -0500 Subject: [PATCH 016/156] chore: reduce indexes on core.tree_access --- packages/core/migrate/incremental/005_tree_access.sql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/core/migrate/incremental/005_tree_access.sql b/packages/core/migrate/incremental/005_tree_access.sql index 4552bc9..2efb1c9 100644 --- a/packages/core/migrate/incremental/005_tree_access.sql +++ b/packages/core/migrate/incremental/005_tree_access.sql @@ -8,8 +8,5 @@ create table core.tree_access , access int not null check (access in (1, 2, 3)) -- 1 = read, 2 = write, 3 = owner , created_at timestamptz not null default now() , updated_at timestamptz -, unique (principal_id, space_id, tree_path) include (access) +, unique (space_id, principal_id, tree_path) include (access) ); - --- list access per space or per space and principal -create index on core.tree_access (space_id, principal_id); From 8605705f79147c489f1b715c8d43426e39435645 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 09:37:45 -0500 Subject: [PATCH 017/156] cover principal name indexes --- packages/core/migrate/incremental/002_principal.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/migrate/incremental/002_principal.sql b/packages/core/migrate/incremental/002_principal.sql index 51eb2e9..360b39c 100644 --- a/packages/core/migrate/incremental/002_principal.sql +++ b/packages/core/migrate/incremental/002_principal.sql @@ -28,8 +28,8 @@ create table core.principal ); -- users must have a globally unique name -create unique index on core.principal (name) where user_id is not null; +create unique index on core.principal (name) include (user_id) where user_id is not null; -- each user's agents must have a unique name (per that user) -create unique index on core.principal (owner_id, name) where agent_id is not null; +create unique index on core.principal (owner_id, name) include (agent_id) where agent_id is not null; -- each space's groups must have a unique name (per that space) create unique index on core.principal (space_id, name) include (group_id) where group_id is not null; From b7a396e2b9c30ba0b824e375d7e85750fa5c2869 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 29 May 2026 09:39:32 -0500 Subject: [PATCH 018/156] simplify principal name indexes --- packages/core/migrate/incremental/002_principal.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/migrate/incremental/002_principal.sql b/packages/core/migrate/incremental/002_principal.sql index 360b39c..7680911 100644 --- a/packages/core/migrate/incremental/002_principal.sql +++ b/packages/core/migrate/incremental/002_principal.sql @@ -30,6 +30,6 @@ create table core.principal -- users must have a globally unique name create unique index on core.principal (name) include (user_id) where user_id is not null; -- each user's agents must have a unique name (per that user) -create unique index on core.principal (owner_id, name) include (agent_id) where agent_id is not null; +create unique index on core.principal (owner_id, name) where agent_id is not null; -- each space's groups must have a unique name (per that space) -create unique index on core.principal (space_id, name) include (group_id) where group_id is not null; +create unique index on core.principal (space_id, name) where group_id is not null; From 649410ee674d11cfd515f168be095fbea7f2881b Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 1 Jun 2026 17:47:14 +0200 Subject: [PATCH 019/156] test: real-DB integration tests for core/space migrations (+ templating, postgres.js pilot) Add integration test suites for the new core/space migration system, run against a real ghost (TigerData) Postgres via TEST_DATABASE_URL. Tests isolate per-schema (core_test_, me_) so they run concurrently and parallel-safe across files; bun run test:db runs core + space. Template the core migration SQL with {{schema}} so tests provision throwaway, isolated cores and never touch a real control plane; production still defaults to 'core' (exposed as CORE_SCHEMA). migrateCore now takes an optional schema. Pilot the Bun.SQL -> postgres.js driver swap on the migrate path. Bun.SQL fails to return a pooled connection after a query/transaction error (oven-sh/bun#22395, present in 1.3.13 and 1.3.14), which hangs the suites and is a latent production hazard for the long-lived engine/accounts pools. Both postgres.js and pg fix it; postgres.js is a near-drop-in. Converted core/space migrate + bootstrap + scripts/migrate-db.ts + test-utils; verified on local and ghost. Docs: CLAUDE.md gains a db-integration-test section and the Bun.SQL -> postgres.js migration recipe; TODO.md tracks test-utils/migration-runner consolidation and the core/space packaging question. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 55 ++++ TODO.md | 58 ++++ bun.lock | 5 + package.json | 1 + packages/core/index.ts | 6 +- .../core/migrate/idempotent/000_update.sql | 24 +- .../idempotent/001_principal_space.sql | 10 +- .../migrate/idempotent/002_group_member.sql | 10 +- .../migrate/idempotent/003_tree_access.sql | 30 +- .../core/migrate/incremental/001_space.sql | 2 +- .../migrate/incremental/002_principal.sql | 12 +- .../incremental/003_principal_space.sql | 8 +- .../migrate/incremental/004_group_member.sql | 10 +- .../migrate/incremental/005_tree_access.sql | 6 +- .../core/migrate/incremental/006_api_key.sql | 4 +- .../core/migrate/migrate.integration.test.ts | 263 +++++++++++++++++ packages/core/migrate/migrate.ts | 117 +++++--- packages/core/migrate/provision.sql | 10 +- packages/core/migrate/test-utils.ts | 202 +++++++++++++ packages/core/package.json | 3 +- packages/space/migrate/bootstrap.ts | 18 +- .../space/migrate/migrate.integration.test.ts | 279 ++++++++++++++++++ packages/space/migrate/migrate.ts | 35 ++- packages/space/migrate/test-utils.ts | 251 ++++++++++++++++ packages/space/package.json | 3 +- packages/space/slug.test.ts | 76 +++++ scripts/migrate-db.ts | 6 +- scripts/package.json | 1 + 28 files changed, 1376 insertions(+), 129 deletions(-) create mode 100644 TODO.md create mode 100644 packages/core/migrate/migrate.integration.test.ts create mode 100644 packages/core/migrate/test-utils.ts create mode 100644 packages/space/migrate/migrate.integration.test.ts create mode 100644 packages/space/migrate/test-utils.ts create mode 100644 packages/space/slug.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 69bd850..f5e0459 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,24 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): **Important**: After making code changes, always run `./bun run check`. +### Database integration tests + +`*.integration.test.ts` files run against a real PostgreSQL 18 with the +required extensions (citext, ltree, pgvector, pg_textsearch), provisioned with +ghost. Point `TEST_DATABASE_URL` at a ghost database and run: + +```bash +TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db +``` + +`testing_me` is the dedicated ghost database for these tests. + +Isolation is **schema-level** (ghost forbids `create database`): each test +provisions its own schema — `core_test_` for core, `me_` for +spaces — so the suites are fully concurrent and parallel-safe across files. +The core migrations are templated so production uses `core` while tests target +throwaway schemas and never touch a real control plane. + ## Style Guides **TypeScript**: Biome for linting and formatting. Config in `biome.json`. @@ -94,3 +112,40 @@ create table me.memory - **Database-native**: Uses PostgreSQL extensions (ltree, pgvector, JSONB GIN, tstzrange, BM25) instead of application-layer abstractions. - **Flexibility over prescription**: `meta` accepts any JSON, `tree` paths are user-defined, `temporal` is optional. No enforced conventions. - **MCP compatibility**: All tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). + +## Database driver migration: Bun.SQL → postgres.js (in progress) + +**Why:** `Bun.SQL` (`new Bun.SQL(...)`) does not return a pooled connection after a +query or `begin()` callback errors — after `max` such errors the pool drains and the +next acquire hangs forever (Bun bug [oven-sh/bun#22395](https://github.com/oven-sh/bun/issues/22395), +reproduced on 1.3.13 and 1.3.14). Any *expected* constraint violation on a long-lived +pool — e.g. the engine/accounts pools in `packages/server/index.ts` — can wedge the +server until restart. Both `postgres` (postgres.js) and `pg` fix it on the Bun runtime; +we use **postgres.js** because `Bun.SQL`'s API was modeled on it, so it's a near-drop-in. + +**Done & verified (local + ghost):** the migrate path — `packages/core/migrate/*`, +`packages/space/migrate/*` (incl. `test-utils.ts`), and `scripts/migrate-db.ts`. + +**Remaining**, package by package, each behind its own integration tests: +`packages/engine` (`db.ts`, `ops/*`, `migrate/*`), `packages/accounts` (`db.ts`, `ops/*`, +`migrate/*`), `packages/server` (`index.ts` pools, `context.ts`, handlers), `packages/worker`. +Spot-check `halfvec`/`ltree`/`tstzrange` round-trips and the `sql(identifier)` interpolations. + +**Per-file recipe:** +- Add `"postgres": "^3.4.9"` to the package's `package.json`. +- `import { SQL } from "bun"` → `import postgres from "postgres"` (value) and/or + `import type { Sql as SQL } from "postgres"` (type). Type a param that receives a + transaction (`sql.begin`'s `tx`) as `ISql<{}>` — both `Sql` and `TransactionSql` extend + `ISql`; keep `Sql<{}>` only for code that calls `.begin`. +- `new Bun.SQL(url, { max, idleTimeout, maxLifetime, connectionTimeout })` → + `postgres(url, { max, idle_timeout, max_lifetime, connect_timeout, onnotice: () => {} })` + (snake_case; `onnotice` silences routine migration NOTICEs). +- `sql.close()` → `sql.end()`. +- `error instanceof SQL.PostgresError` → duck-type (`(error as { position?: unknown }).position`). +- Rows: postgres.js returns a typed `Row` (index signature), but `noUncheckedIndexedAccess` + makes `rows[0]` possibly-`undefined` → `const [row] = ...; row?.col`, and drop + `(r: { col: T })` annotations on `.map` callbacks (`r` is `Row`). + +**Test gotcha:** `expect(sql\`…\`).rejects` **hangs** in bun:test — it doesn't drive +postgres.js's lazy `PendingQuery`. Assert query failures with try/catch (see `expectReject` +in `migrate/test-utils.ts`). `expect(migrateX(…)).rejects` is fine (real async-fn Promise). diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..f2a0edd --- /dev/null +++ b/TODO.md @@ -0,0 +1,58 @@ +# TODO + +Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, +see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). The two +consolidations below are best done as part of that rollout, since it touches every +package's migration and test code anyway. + +## Open question: should `core` and `space` be one package? + +They're separate packages today (`packages/core`, `packages/space`) with no runtime +dependency between them. The consolidations below keep bumping into that split — +sharing code requires a separate shared/dev package rather than a plain internal +module. So: should they merge into one package? + +- **Merge** → sharing the test-utils and migration runner becomes trivial (internal + modules, no extra package); they're conceptually the two halves (control plane + + data plane) of one system. +- **Keep separate** → preserves a clean boundary (space already has zero references to + core) and keeps the door open to deploying/scaling them differently (e.g. space + schemas sharded across many DBs via pgdog, core centralized). + +Decide this first — it determines whether the consolidations below land as internal +modules (merged) or a shared package (separate). + +## Consolidate duplicated test-utils + +`packages/core/migrate/test-utils.ts` and `packages/space/migrate/test-utils.ts` +duplicate ~110 lines of generic, driver-level helpers: `resolveTestDatabaseUrl`, +`connect`, `expectReject`, and schema introspection (`schemaExists`, `tableExists`, +`listTables`, `listFunctions`, `listTriggers`, `appliedMigrations`, +`getSchemaVersion`). `packages/engine` and `packages/accounts` also carry their own +copies of some of these (~4 repo-wide copies of `tableExists`/`schemaExists`). + +- [ ] Extract the generic helpers into a shared **dev-only** package (e.g. + `@memory.build/db-testkit`) added as a `devDependency` where needed. Keep + package-specific provisioning in each package: `TestCore`/`TestSpace`, + `randomCoreSchema`/`randomSlug`, `columnType`, the index helpers. Test-only, + so it doesn't couple the packages at runtime. +- [ ] Move `engine`/`accounts` test-utils onto it too, removing the older duplicates. + +## Consolidate the migration runner logic + +`packages/core/migrate/migrate.ts`, `packages/space/migrate/migrate.ts`, and +`packages/space/migrate/bootstrap.ts` duplicate most of the migration machinery: + +- advisory locking (`advisoryLockKey`, `acquireAdvisoryLock`, retry/backoff) +- SQL-file execution with error-location logging (`executeSqlFile`, + `logPostgresSqlLocation`, `sqlLocation`, `sqlContext`) +- extension / Postgres-version preconditions (`ensureExtension`, + `ensurePostgresVersion`, `REQUIRED_EXTENSIONS`) +- `{{template}}` rendering +- the incremental-once / idempotent-always runner, with version + migration tracking +- telemetry span wrapping + +- [ ] Extract this into a shared migration util (e.g. `@memory.build/migrate-kit`) + parameterized by schema name, the ordered incremental/idempotent SQL lists, and + template vars. `migrateCore` / `migrateSpace` / `bootstrapSpaceDatabase` then + become thin callers, leaving each `migrate.ts` with only its schema-specific bits. diff --git a/bun.lock b/bun.lock index 6f341f7..ec47cfe 100644 --- a/bun.lock +++ b/bun.lock @@ -54,6 +54,7 @@ "version": "0.2.5", "dependencies": { "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", }, }, "packages/docs-site": { @@ -135,6 +136,7 @@ "version": "0.2.5", "dependencies": { "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", }, }, "packages/web": { @@ -180,6 +182,7 @@ "@memory.build/core": "workspace:*", "@memory.build/embedding": "workspace:*", "@memory.build/space": "workspace:*", + "postgres": "^3.4.9", "yaml": "^2.7.0", }, }, @@ -1225,6 +1228,8 @@ "postcss": ["postcss@8.5.10", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ=="], + "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="], + "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="], diff --git a/package.json b/package.json index 7897c2a..2741833 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "server": "./bun run packages/server/index.ts", "setup": "./bun scripts/setup.ts", "test": "./bun test packages", + "test:db": "find packages/core packages/space -name '*.integration.test.ts' -print0 | xargs -0 -P 4 -n 1 ./bun test --timeout 30000", "typecheck": "tsc --noEmit" }, "devDependencies": { diff --git a/packages/core/index.ts b/packages/core/index.ts index fc637ec..8dd575c 100644 --- a/packages/core/index.ts +++ b/packages/core/index.ts @@ -1,2 +1,6 @@ -export { type MigrateCoreOptions, migrateCore } from "./migrate/migrate"; +export { + CORE_SCHEMA, + type MigrateCoreOptions, + migrateCore, +} from "./migrate/migrate"; export { CORE_SCHEMA_VERSION } from "./version"; diff --git a/packages/core/migrate/idempotent/000_update.sql b/packages/core/migrate/idempotent/000_update.sql index 53a4ce6..16fc5c7 100644 --- a/packages/core/migrate/idempotent/000_update.sql +++ b/packages/core/migrate/idempotent/000_update.sql @@ -1,5 +1,5 @@ -- generic trigger function to update updated_at timestamp -create or replace function core.update_updated_at() +create or replace function {{schema}}.update_updated_at() returns trigger as $func$ begin @@ -7,29 +7,29 @@ begin return new; end; $func$ language plpgsql volatile security definer -set search_path to core, pg_temp; +set search_path to {{schema}}, pg_temp; create or replace trigger space_before_update_trg -before update on core.space +before update on {{schema}}.space for each row -execute function core.update_updated_at(); +execute function {{schema}}.update_updated_at(); create or replace trigger principal_before_update_trg -before update on core.principal +before update on {{schema}}.principal for each row -execute function core.update_updated_at(); +execute function {{schema}}.update_updated_at(); create or replace trigger principal_space_before_update_trg -before update on core.principal_space +before update on {{schema}}.principal_space for each row -execute function core.update_updated_at(); +execute function {{schema}}.update_updated_at(); create or replace trigger group_member_before_update_trg -before update on core.group_member +before update on {{schema}}.group_member for each row -execute function core.update_updated_at(); +execute function {{schema}}.update_updated_at(); create or replace trigger tree_access_before_update_trg -before update on core.tree_access +before update on {{schema}}.tree_access for each row -execute function core.update_updated_at(); +execute function {{schema}}.update_updated_at(); diff --git a/packages/core/migrate/idempotent/001_principal_space.sql b/packages/core/migrate/idempotent/001_principal_space.sql index b221c4b..e939a6d 100644 --- a/packages/core/migrate/idempotent/001_principal_space.sql +++ b/packages/core/migrate/idempotent/001_principal_space.sql @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------- -- is_principal_in_space ------------------------------------------------------------------------------- -create or replace function core.is_principal_in_space +create or replace function {{schema}}.is_principal_in_space ( _principal_id uuid , _space_id uuid ) @@ -10,7 +10,7 @@ as $func$ select exists ( select 1 - from core.principal_space ps + from {{schema}}.principal_space ps where ps.principal_id = _principal_id and ps.space_id = _space_id ) @@ -20,7 +20,7 @@ $func$ language sql stable security invoker ------------------------------------------------------------------------------- -- is_principal_space_admin ------------------------------------------------------------------------------- -create or replace function core.is_principal_space_admin +create or replace function {{schema}}.is_principal_space_admin ( _principal_id uuid , _space_id uuid ) @@ -30,8 +30,8 @@ as $func$ ( ( select ps.admin and (not p.kind = 'a') -- agents cannot be space admins - from core.principal_space ps - inner join core.principal p on (ps.principal_id = p.id) + from {{schema}}.principal_space ps + inner join {{schema}}.principal p on (ps.principal_id = p.id) where ps.principal_id = _principal_id and ps.space_id = _space_id ) diff --git a/packages/core/migrate/idempotent/002_group_member.sql b/packages/core/migrate/idempotent/002_group_member.sql index f7e32c7..b2c7a8b 100644 --- a/packages/core/migrate/idempotent/002_group_member.sql +++ b/packages/core/migrate/idempotent/002_group_member.sql @@ -2,7 +2,7 @@ ------------------------------------------------------------------------------- -- member_groups ------------------------------------------------------------------------------- -create or replace function core.member_groups +create or replace function {{schema}}.member_groups ( _member_id uuid , _space_id uuid ) @@ -14,13 +14,13 @@ as $func$ select gm.group_id , gm.admin and (not m.kind = 'a') -- agent's cannot be group admins - from core.principal m -- the member + from {{schema}}.principal m -- the member -- assert the member belongs to the space - inner join core.principal_space psm on (m.id = psm.principal_id and psm.space_id = _space_id) + inner join {{schema}}.principal_space psm on (m.id = psm.principal_id and psm.space_id = _space_id) -- find the groups the member belongs to in the space - inner join core.group_member gm on (m.member_id = gm.member_id and gm.space_id = _space_id) + inner join {{schema}}.group_member gm on (m.member_id = gm.member_id and gm.space_id = _space_id) -- assert the group belongs to the space - inner join core.principal_space psg on (gm.group_id = psg.principal_id and psg.space_id = _space_id) + inner join {{schema}}.principal_space psg on (gm.group_id = psg.principal_id and psg.space_id = _space_id) where m.member_id = _member_id -- the member $func$ language sql stable security invoker ; diff --git a/packages/core/migrate/idempotent/003_tree_access.sql b/packages/core/migrate/idempotent/003_tree_access.sql index ad804a5..7dfe20f 100644 --- a/packages/core/migrate/idempotent/003_tree_access.sql +++ b/packages/core/migrate/idempotent/003_tree_access.sql @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------- -- member_tree_access ------------------------------------------------------------------------------- -create or replace function core.member_tree_access +create or replace function {{schema}}.member_tree_access ( _member_id uuid , _space_id uuid ) @@ -14,15 +14,15 @@ as $func$ select ta.tree_path , ta.access - from core.member_groups(_member_id, _space_id) mg - inner join core.tree_access ta on (mg.group_id = ta.principal_id and ta.space_id = _space_id) + from {{schema}}.member_groups(_member_id, _space_id) mg + inner join {{schema}}.tree_access ta on (mg.group_id = ta.principal_id and ta.space_id = _space_id) $func$ language sql stable security invoker ; ------------------------------------------------------------------------------- -- user_tree_access ------------------------------------------------------------------------------- -create or replace function core.user_tree_access +create or replace function {{schema}}.user_tree_access ( _user_id uuid , _space_id uuid ) @@ -35,23 +35,23 @@ as $func$ select ta.tree_path , ta.access - from core.principal u - inner join core.principal_space psu on (u.id = psu.principal_id and psu.space_id = _space_id) - inner join core.tree_access ta on (u.id = ta.principal_id and ta.space_id = _space_id) + from {{schema}}.principal u + inner join {{schema}}.principal_space psu on (u.id = psu.principal_id and psu.space_id = _space_id) + inner join {{schema}}.tree_access ta on (u.id = ta.principal_id and ta.space_id = _space_id) where u.user_id = _user_id union -- user's access via groups select x.tree_path , x.access - from core.member_tree_access(_user_id, _space_id) x + from {{schema}}.member_tree_access(_user_id, _space_id) x $func$ language sql stable security invoker ; ------------------------------------------------------------------------------- -- agent_tree_access ------------------------------------------------------------------------------- -create or replace function core.agent_tree_access +create or replace function {{schema}}.agent_tree_access ( _agent_id uuid , _space_id uuid ) @@ -66,16 +66,16 @@ as $func$ select ta.tree_path , ta.access - from core.principal a - inner join core.principal_space ps on (a.id = ps.principal_id and ps.space_id = _space_id) - inner join core.tree_access ta on (a.id = ta.principal_id and ta.space_id = _space_id) + from {{schema}}.principal a + inner join {{schema}}.principal_space ps on (a.id = ps.principal_id and ps.space_id = _space_id) + inner join {{schema}}.tree_access ta on (a.id = ta.principal_id and ta.space_id = _space_id) where a.agent_id = _agent_id union -- agent's access via groups select x.tree_path , x.access - from core.member_tree_access(_agent_id, _space_id) x + from {{schema}}.member_tree_access(_agent_id, _space_id) x ) , owner_access as materialized ( @@ -86,10 +86,10 @@ as $func$ from ( select p.owner_id - from core.principal p + from {{schema}}.principal p where p.agent_id = _agent_id ) a - cross join lateral core.user_tree_access(a.owner_id, _space_id) x + cross join lateral {{schema}}.user_tree_access(a.owner_id, _space_id) x ) select x.tree_path diff --git a/packages/core/migrate/incremental/001_space.sql b/packages/core/migrate/incremental/001_space.sql index 931d2f7..a8ea560 100644 --- a/packages/core/migrate/incremental/001_space.sql +++ b/packages/core/migrate/incremental/001_space.sql @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------- -- space ------------------------------------------------------------------------------- -create table core.space +create table {{schema}}.space ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) , slug text not null unique check (slug ~ '^[a-z0-9]{12}$') , name citext not null diff --git a/packages/core/migrate/incremental/002_principal.sql b/packages/core/migrate/incremental/002_principal.sql index 7680911..b5c590e 100644 --- a/packages/core/migrate/incremental/002_principal.sql +++ b/packages/core/migrate/incremental/002_principal.sql @@ -1,14 +1,14 @@ ------------------------------------------------------------------------------- -- principal ------------------------------------------------------------------------------- -create table core.principal +create table {{schema}}.principal ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) , user_id uuid unique nulls distinct generated always as (case when kind = 'u' then id else null end) stored , group_id uuid unique nulls distinct generated always as (case when kind = 'g' then id else null end) stored , agent_id uuid unique nulls distinct generated always as (case when kind = 'a' then id else null end) stored , member_id uuid unique nulls distinct generated always as (case when kind in ('u', 'a') then id else null end) stored -, owner_id uuid references core.principal (user_id) on delete cascade -- points to agent's owner -, space_id uuid references core.space (id) on delete cascade +, owner_id uuid references {{schema}}.principal (user_id) on delete cascade -- points to agent's owner +, space_id uuid references {{schema}}.space (id) on delete cascade , kind text not null check (kind in ('g', 'u', 'a')) -- group, user, agent , name citext not null check (name::text !~ '/') -- agent names are displayed as / , created_at timestamptz not null default now() @@ -28,8 +28,8 @@ create table core.principal ); -- users must have a globally unique name -create unique index on core.principal (name) include (user_id) where user_id is not null; +create unique index on {{schema}}.principal (name) include (user_id) where user_id is not null; -- each user's agents must have a unique name (per that user) -create unique index on core.principal (owner_id, name) where agent_id is not null; +create unique index on {{schema}}.principal (owner_id, name) where agent_id is not null; -- each space's groups must have a unique name (per that space) -create unique index on core.principal (space_id, name) where group_id is not null; +create unique index on {{schema}}.principal (space_id, name) where group_id is not null; diff --git a/packages/core/migrate/incremental/003_principal_space.sql b/packages/core/migrate/incremental/003_principal_space.sql index a00b996..ede33e0 100644 --- a/packages/core/migrate/incremental/003_principal_space.sql +++ b/packages/core/migrate/incremental/003_principal_space.sql @@ -1,13 +1,13 @@ ------------------------------------------------------------------------------- -- principal_space ------------------------------------------------------------------------------- -create table core.principal_space -( space_id uuid not null references core.space (id) on delete cascade -, principal_id uuid not null references core.principal (id) on delete cascade -- can be users, agents, or groups +create table {{schema}}.principal_space +( space_id uuid not null references {{schema}}.space (id) on delete cascade +, principal_id uuid not null references {{schema}}.principal (id) on delete cascade -- can be users, agents, or groups , admin bool not null default false , created_at timestamptz not null default now() , updated_at timestamptz , unique (principal_id, space_id) include (admin) ); -create index on core.principal_space (space_id, principal_id) include (admin); +create index on {{schema}}.principal_space (space_id, principal_id) include (admin); diff --git a/packages/core/migrate/incremental/004_group_member.sql b/packages/core/migrate/incremental/004_group_member.sql index 4821cf6..e29cc50 100644 --- a/packages/core/migrate/incremental/004_group_member.sql +++ b/packages/core/migrate/incremental/004_group_member.sql @@ -1,10 +1,10 @@ ------------------------------------------------------------------------------- -- group_member ------------------------------------------------------------------------------- -create table core.group_member -( space_id uuid not null references core.space (id) on delete cascade -, group_id uuid not null references core.principal (group_id) on delete cascade -- can only be groups -, member_id uuid not null references core.principal (member_id) on delete cascade -- can be users or agents, but not groups +create table {{schema}}.group_member +( space_id uuid not null references {{schema}}.space (id) on delete cascade +, group_id uuid not null references {{schema}}.principal (group_id) on delete cascade -- can only be groups +, member_id uuid not null references {{schema}}.principal (member_id) on delete cascade -- can be users or agents, but not groups , admin bool not null default false , created_at timestamptz not null default now() , updated_at timestamptz @@ -12,4 +12,4 @@ create table core.group_member ); -- index for listing groups in a space and/or members of a group -create index on core.group_member (space_id, group_id, member_id) include (admin); +create index on {{schema}}.group_member (space_id, group_id, member_id) include (admin); diff --git a/packages/core/migrate/incremental/005_tree_access.sql b/packages/core/migrate/incremental/005_tree_access.sql index 2efb1c9..2bf4114 100644 --- a/packages/core/migrate/incremental/005_tree_access.sql +++ b/packages/core/migrate/incremental/005_tree_access.sql @@ -1,9 +1,9 @@ ------------------------------------------------------------------------------- -- tree_access ------------------------------------------------------------------------------- -create table core.tree_access -( space_id uuid not null references core.space (id) on delete cascade -, principal_id uuid not null references core.principal (id) on delete cascade -- can be users, agents, or groups +create table {{schema}}.tree_access +( space_id uuid not null references {{schema}}.space (id) on delete cascade +, principal_id uuid not null references {{schema}}.principal (id) on delete cascade -- can be users, agents, or groups , tree_path ltree not null , access int not null check (access in (1, 2, 3)) -- 1 = read, 2 = write, 3 = owner , created_at timestamptz not null default now() diff --git a/packages/core/migrate/incremental/006_api_key.sql b/packages/core/migrate/incremental/006_api_key.sql index 85b7250..8d6418f 100644 --- a/packages/core/migrate/incremental/006_api_key.sql +++ b/packages/core/migrate/incremental/006_api_key.sql @@ -1,9 +1,9 @@ ------------------------------------------------------------------------------- -- api_key ------------------------------------------------------------------------------- -create table core.api_key +create table {{schema}}.api_key ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) -, member_id uuid not null references core.principal (member_id) on delete cascade -- may be users or agents, not groups +, member_id uuid not null references {{schema}}.principal (member_id) on delete cascade -- may be users or agents, not groups , lookup_id text unique not null check (lookup_id ~ '^[A-Za-z0-9_-]{16}$') , secret text not null -- hashed secret , name text not null diff --git a/packages/core/migrate/migrate.integration.test.ts b/packages/core/migrate/migrate.integration.test.ts new file mode 100644 index 0000000..83e5c03 --- /dev/null +++ b/packages/core/migrate/migrate.integration.test.ts @@ -0,0 +1,263 @@ +// Integration tests for the `core` control-plane migrations (migrateCore). +// +// The core migrations are templated, so each test targets its own throwaway +// `core_test_` schema — never the real `core`. That makes these tests +// isolated and safe to run against any database (including a shared dev one). +// Read-only shape assertions share one canonical core provisioned in beforeAll; +// the few behavior tests provision their own. Tests run serially within the +// file; cross-suite parallelism comes from `bun run test:db` (separate +// processes for core and space). +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Sql as SQL } from "postgres"; +import { CORE_SCHEMA_VERSION } from "../version"; +import { migrateCore } from "./migrate"; +import { + appliedMigrations, + connect, + expectReject, + extensionInstalled, + getSchemaVersion, + listFunctions, + listTables, + listTriggers, + randomCoreSchema, + schemaExists, + TestCore, + tableExists, + withTestCore, +} from "./test-utils"; + +const EXPECTED_TABLES = [ + "api_key", + "group_member", + "migration", + "principal", + "principal_space", + "space", + "tree_access", + "version", +]; + +const EXPECTED_MIGRATIONS = [ + "001_space", + "002_principal", + "003_principal_space", + "004_group_member", + "005_tree_access", + "006_api_key", +]; + +const EXPECTED_FUNCTIONS = [ + "agent_tree_access", + "is_principal_in_space", + "is_principal_space_admin", + "member_groups", + "member_tree_access", + "update_updated_at", + "user_tree_access", +]; + +const REQUIRED_EXTENSIONS = ["citext", "ltree", "vector", "pg_textsearch"]; + +let sql: SQL; +// One migrated core shared by all read-only shape/function assertions. +let canonical: TestCore; + +beforeAll(async () => { + sql = connect(12); + canonical = await TestCore.create(sql); // migrateCore installs extensions itself +}); + +afterAll(async () => { + await canonical?.drop(); + await sql.end(); +}); + +describe("provisioned core schema", () => { + test("provisions into the requested (templated) schema", async () => { + expect(canonical.schema).toMatch(/^core_test_/); + expect(await schemaExists(sql, canonical.schema)).toBe(true); + }); + + test("creates infrastructure and domain tables", async () => { + const tables = await listTables(sql, canonical.schema); + for (const table of EXPECTED_TABLES) { + expect(tables).toContain(table); + } + }); + + test("records every incremental migration exactly once", async () => { + expect(await appliedMigrations(sql, canonical.schema)).toEqual( + EXPECTED_MIGRATIONS, + ); + }); + + test("stamps the schema version", async () => { + expect(await getSchemaVersion(sql, canonical.schema)).toBe( + CORE_SCHEMA_VERSION, + ); + }); + + test("installs all required extensions", async () => { + for (const ext of REQUIRED_EXTENSIONS) { + expect(await extensionInstalled(sql, ext)).toBe(true); + } + }); + + test("creates the access-control functions in the schema", async () => { + const functions = await listFunctions(sql, canonical.schema); + for (const fn of EXPECTED_FUNCTIONS) { + expect(functions).toContain(fn); + } + }); + + test("installs updated_at triggers on mutable tables", async () => { + for (const table of [ + "space", + "principal", + "principal_space", + "group_member", + "tree_access", + ]) { + const triggers = await listTriggers(sql, canonical.schema, table); + expect(triggers).toContain(`${table}_before_update_trg`); + } + }); +}); + +describe("schema constraints enforce", () => { + test("principal.kind is restricted to g/u/a", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.principal (kind, name) values ('x', 'bad-kind')`, + ), + ); + }); + + test("principal ids must be UUIDv7", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.principal (id, kind, name) + values ('00000000-0000-4000-8000-000000000000', 'u', 'v4-id')`, + ), + ); + }); + + test("space.slug must be 12 lowercase alphanumerics", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.space (slug, name) values ('BAD', 'x')`, + ), + ); + }); + + test("user names are globally unique", async () => { + const name = `smoke_unique_${crypto.randomUUID().slice(0, 8)}`; + await sql.unsafe( + `insert into ${canonical.schema}.principal (kind, name) values ('u', '${name}')`, + ); + try { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.principal (kind, name) values ('u', '${name}')`, + ), + ); + } finally { + await sql.unsafe( + `delete from ${canonical.schema}.principal where name = '${name}'`, + ); + } + }); +}); + +describe("access-control functions are callable", () => { + // Catches functions that "exist" but reference missing columns/types: a bad + // body only errors when executed, not when created. + const dummy = "00000000-0000-7000-8000-000000000000"; + + test("access functions execute against empty data", async () => { + const s = canonical.schema; + await sql.unsafe( + `select * from ${s}.user_tree_access('${dummy}', '${dummy}')`, + ); + await sql.unsafe( + `select * from ${s}.agent_tree_access('${dummy}', '${dummy}')`, + ); + await sql.unsafe( + `select * from ${s}.member_tree_access('${dummy}', '${dummy}')`, + ); + await sql.unsafe( + `select * from ${s}.member_groups('${dummy}', '${dummy}')`, + ); + }); + + test("predicate functions return false for unknown principals", async () => { + const s = canonical.schema; + const [a] = await sql.unsafe( + `select ${s}.is_principal_in_space('${dummy}', '${dummy}') as v`, + ); + expect(a?.v).toBe(false); + const [b] = await sql.unsafe( + `select ${s}.is_principal_space_admin('${dummy}', '${dummy}') as v`, + ); + expect(b?.v).toBe(false); + }); +}); + +describe("migration behavior", () => { + test("is idempotent: re-running changes no migration rows or version", async () => { + await withTestCore(sql, {}, async (core) => { + const before = await appliedMigrations(sql, core.schema); + await migrateCore(sql, { schema: core.schema }); + expect(await appliedMigrations(sql, core.schema)).toEqual(before); + expect(await getSchemaVersion(sql, core.schema)).toBe( + CORE_SCHEMA_VERSION, + ); + }); + }); + + test("rejects a downgrade (db version newer than app)", async () => { + await withTestCore(sql, {}, async (core) => { + await sql.unsafe(`update ${core.schema}.version set version = '99.0.0'`); + await expect(migrateCore(sql, { schema: core.schema })).rejects.toThrow( + /older than database version/, + ); + }); + }); + + test("rejects invalid schema names", async () => { + for (const schema of ["Bad-Schema", "1core", "core test", "core;drop"]) { + await expect(migrateCore(sql, { schema })).rejects.toThrow( + /Invalid core schema name/, + ); + } + }); + + test("concurrent migrateCore on one schema is serialized safely", async () => { + // The advisory lock serializes writers. A loser may exhaust its retry + // budget and throw "Unable to acquire lock" — expected, not corruption. + // What must hold: at least one succeeds and the schema stays valid. + const schema = randomCoreSchema(); + try { + const results = await Promise.allSettled([ + migrateCore(sql, { schema }), + migrateCore(sql, { schema }), + migrateCore(sql, { schema }), + ]); + + expect(results.some((r) => r.status === "fulfilled")).toBe(true); + for (const r of results) { + if (r.status === "rejected") { + expect(String((r.reason as Error)?.message ?? r.reason)).toContain( + "Unable to acquire lock", + ); + } + } + + expect(await getSchemaVersion(sql, schema)).toBe(CORE_SCHEMA_VERSION); + expect(await tableExists(sql, schema, "principal")).toBe(true); + } finally { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + } + }); +}); diff --git a/packages/core/migrate/migrate.ts b/packages/core/migrate/migrate.ts index c9e5fe0..204d130 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/core/migrate/migrate.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; -import { SQL, semver } from "bun"; +import { semver } from "bun"; +import type { ISql, Sql as SQL } from "postgres"; import { CORE_SCHEMA_VERSION } from "../version"; import incremental001 from "./incremental/001_space.sql" with { type: "text" }; import incremental002 from "./incremental/002_principal.sql" with { @@ -91,7 +92,14 @@ const idempotents: Idempotent[] = [ }, ]; -const CORE_SCHEMA = "core"; +/** + * The core control-plane schema name. Production always uses "core"; the name + * is a parameter so tests can provision throwaway, isolated cores (and so the + * SQL is templated symmetrically with the per-space migrations). Reference this + * constant rather than hardcoding "core" elsewhere. + */ +export const CORE_SCHEMA = "core"; + const REQUIRED_EXTENSIONS = [ { name: "citext", minVersion: "1.6" }, { name: "ltree", minVersion: "1.3" }, @@ -100,6 +108,7 @@ const REQUIRED_EXTENSIONS = [ ] as const; export interface MigrateCoreOptions { + schema?: string; logSqlFiles?: boolean; statementTimeout?: string; lockTimeout?: string; @@ -108,6 +117,7 @@ export interface MigrateCoreOptions { } interface NormalizedMigrateCoreOptions { + schema: string; logSqlFiles: boolean; schemaVersion: string; statementTimeout: string; @@ -127,10 +137,17 @@ export async function migrateCore( attributes, callback: async () => { try { + if (!isValidSchemaName(opts.schema)) { + throw new Error( + `Invalid core schema name: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, + ); + } if (!semver.satisfies(opts.schemaVersion, "*")) { throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); } - const [key1, key2] = advisoryLockKey("memory-core:schema:core"); + const [key1, key2] = advisoryLockKey( + `memory-core:schema:${opts.schema}`, + ); await sql.begin(async (tx) => { await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; @@ -157,7 +174,7 @@ export async function migrateCore( }); } - if (!(await doesCoreExist(tx))) { + if (!(await doesCoreExist(tx, opts.schema))) { await span("core.migrate.provision", { attributes: { ...attributes, @@ -186,7 +203,7 @@ function migrateAttributes( options: NormalizedMigrateCoreOptions, ): Record { return { - "db.schema": CORE_SCHEMA, + "db.schema": options.schema, "core.schema_version": options.schemaVersion, "core.required_extensions": REQUIRED_EXTENSIONS.map( (extension) => `${extension.name}@>=${extension.minVersion}`, @@ -203,6 +220,7 @@ function normalizeMigrateCoreOptions( options: MigrateCoreOptions, ): NormalizedMigrateCoreOptions { return { + schema: options.schema ?? CORE_SCHEMA, logSqlFiles: options.logSqlFiles ?? false, schemaVersion: CORE_SCHEMA_VERSION, statementTimeout: options.statementTimeout ?? "20s", @@ -213,6 +231,10 @@ function normalizeMigrateCoreOptions( }; } +function isValidSchemaName(schema: string): boolean { + return /^[a-z_][a-z0-9_]*$/.test(schema) && schema.length <= 63; +} + function advisoryLockKey(schema: string): [number, number] { const digest = createHash("sha256").update(schema).digest(); return [digest.readInt32BE(0), digest.readInt32BE(4)]; @@ -226,7 +248,7 @@ function sleep(ms: number): Promise { } async function acquireAdvisoryLock( - tx: SQL, + tx: ISql<{}>, key1: number, key2: number, ): Promise { @@ -235,7 +257,7 @@ async function acquireAdvisoryLock( const [result] = await tx` select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired `; - if (result.acquired) { + if (result?.acquired) { acquired = true; break; } @@ -246,29 +268,36 @@ async function acquireAdvisoryLock( return acquired; } -async function doesCoreExist(tx: SQL): Promise { - const [{ coreExists }] = await tx` +async function doesCoreExist(tx: ISql<{}>, schema: string): Promise { + const [row] = await tx` select exists ( select 1 from pg_namespace n - where n.nspname = ${CORE_SCHEMA} + where n.nspname = ${schema} ) as "coreExists" `; - return coreExists; + return Boolean(row?.coreExists); } async function provisionCore( - tx: SQL, + tx: ISql<{}>, options: NormalizedMigrateCoreOptions, ): Promise { - await executeSqlFile(tx, options, "provision", "provision.sql", provisionSql); + await executeSqlFile( + tx, + options, + "provision", + "provision.sql", + template(provisionSql, { schema: options.schema }), + ); } -async function ensurePostgresVersion(tx: SQL): Promise { - const [{ server_version_num }] = await tx` +async function ensurePostgresVersion(tx: ISql<{}>): Promise { + const [row] = await tx` select current_setting('server_version_num')::int as server_version_num `; + const server_version_num = Number(row?.server_version_num); if (server_version_num < 180000) { throw new Error( `PostgreSQL version 18 or higher is required (found ${server_version_num})`, @@ -277,7 +306,7 @@ async function ensurePostgresVersion(tx: SQL): Promise { } async function ensureExtension( - tx: SQL, + tx: ISql<{}>, name: string, minVersion: string, ): Promise { @@ -319,14 +348,16 @@ async function ensureExtension( } async function runMigrations( - tx: SQL, + tx: ISql<{}>, options: NormalizedMigrateCoreOptions, ): Promise { - await assertSchemaOwnership(tx); + const schema = options.schema; + await assertSchemaOwnership(tx, schema); - const [{ version: dbVersion }] = await tx` - select version from core.version + const [versionRow] = await tx` + select version from ${tx(schema)}.version `; + const dbVersion: string = versionRow?.version; const cmp = semver.order(options.schemaVersion, dbVersion); if (cmp < 0) { throw new Error( @@ -337,7 +368,7 @@ async function runMigrations( /* run migrations regardless if (cmp === 0) { info("Core migration skipped, version current", { - "db.schema": CORE_SCHEMA, + "db.schema": schema, "core.version": dbVersion, "core.schema_version": options.schemaVersion, }); @@ -350,22 +381,22 @@ async function runMigrations( ); for (const migration of sorted1) { - const [{ existing }] = await tx` + const [existingRow] = await tx` select exists ( select 1 - from core.migration + from ${tx(schema)}.migration where name = ${migration.name} ) as existing `; - if (existing) { + if (existingRow?.existing) { continue; } await span("core.migrate.incremental", { attributes: { - "db.schema": CORE_SCHEMA, + "db.schema": schema, "core.migration": migration.name, "core.migration_file": migration.file, "core.migration_type": "incremental", @@ -377,15 +408,15 @@ async function runMigrations( options, "incremental", migration.file, - migration.sql, + template(migration.sql, { schema }), ); await tx` - insert into core.migration (name, applied_at_version) + insert into ${tx(schema)}.migration (name, applied_at_version) values (${migration.name}, ${options.schemaVersion})`; }, }); info("Core migration applied", { - "db.schema": CORE_SCHEMA, + "db.schema": schema, "core.migration": migration.name, "core.migration_file": migration.file, "core.migration_type": "incremental", @@ -398,7 +429,7 @@ async function runMigrations( for (const migration of sorted2) { await span("core.migrate.idempotent", { attributes: { - "db.schema": CORE_SCHEMA, + "db.schema": schema, "core.migration": migration.name, "core.migration_file": migration.file, "core.migration_type": "idempotent", @@ -410,16 +441,16 @@ async function runMigrations( options, "idempotent", migration.file, - migration.sql, + template(migration.sql, { schema }), ), }); } - await tx`update core.version set version = ${options.schemaVersion}, at = now()`; + await tx`update ${tx(schema)}.version set version = ${options.schemaVersion}, at = now()`; } async function executeSqlFile( - tx: SQL, + tx: ISql<{}>, options: NormalizedMigrateCoreOptions, type: string, file: string, @@ -458,8 +489,8 @@ function logSqlExecutionError( } function logPostgresSqlLocation(sqlText: string, error: unknown): void { - if (!(error instanceof SQL.PostgresError)) return; - const position = Number(error.position); + // postgres-js sets `position` (1-based) on server errors; non-PG errors won't. + const position = Number((error as { position?: unknown })?.position); if (!Number.isSafeInteger(position) || position < 1) return; const location = sqlLocation(sqlText, position); @@ -506,17 +537,29 @@ function sqlContext(sqlText: string, line: number, column: number): string { return output.join("\n"); } -async function assertSchemaOwnership(tx: SQL): Promise { +async function assertSchemaOwnership( + tx: ISql<{}>, + schema: string, +): Promise { const [result] = await tx` select n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner from pg_catalog.pg_namespace n - where n.nspname = ${CORE_SCHEMA} + where n.nspname = ${schema} `; if (!result?.is_owner) { throw new Error( - "Only the owner of the core schema can run database migrations", + `Only the owner of the ${schema} schema can run database migrations`, ); } } + +function template(sql: string, vars: Record): string { + return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { + if (!(key in vars)) { + throw new Error(`Missing template variable: ${key}`); + } + return String(vars[key]); + }); +} diff --git a/packages/core/migrate/provision.sql b/packages/core/migrate/provision.sql index 479b55c..e98b9d9 100644 --- a/packages/core/migrate/provision.sql +++ b/packages/core/migrate/provision.sql @@ -1,14 +1,14 @@ -create schema core; +create schema {{schema}}; -create table core.version +create table {{schema}}.version ( version text not null , at timestamptz not null default now() ); -create unique index version_singleton_idx on core.version ((true)); -- only ONE row allowed -insert into core.version (version) values ('0.0.0'); +create unique index version_singleton_idx on {{schema}}.version ((true)); -- only ONE row allowed +insert into {{schema}}.version (version) values ('0.0.0'); -create table core.migration +create table {{schema}}.migration ( name text not null constraint migration_pkey primary key , applied_at_version text not null , applied_at timestamptz not null default pg_catalog.clock_timestamp() diff --git a/packages/core/migrate/test-utils.ts b/packages/core/migrate/test-utils.ts new file mode 100644 index 0000000..ca76a36 --- /dev/null +++ b/packages/core/migrate/test-utils.ts @@ -0,0 +1,202 @@ +import type { Sql as SQL } from "postgres"; +import postgres from "postgres"; +import { type MigrateCoreOptions, migrateCore } from "./migrate"; + +// --------------------------------------------------------------------------- +// Connection +// --------------------------------------------------------------------------- +// +// Tests run against a real Postgres that has the required extensions +// (citext, ltree, vector, pg_textsearch) and is PG 18+. Two ways to provide +// one: +// +// - local docker (fast iteration): ./bun run pg (then leave TEST_DATABASE_URL unset) +// - ghost (real TigerData stack): TEST_DATABASE_URL="$(ghost connect testing_me)" +// +// Because the core migrations are templated (production uses the "core" +// schema; tests pass a unique schema name), every test can provision its own +// throwaway core and run concurrently — exactly like space tests, and without +// ever touching a real `core` schema. + +const DEFAULT_TEST_DATABASE_URL = + "postgresql://postgres@127.0.0.1:5432/postgres"; + +export function resolveTestDatabaseUrl(): string { + return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; +} + +export function connect(max = 10): SQL { + // onnotice silences the routine "… already exists, skipping" NOTICEs that the + // idempotent migrations emit (postgres-js prints them to the console by default). + return postgres(resolveTestDatabaseUrl(), { max, onnotice: () => {} }); +} + +const SCHEMA_SUFFIX_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; + +/** A unique, valid core schema name, e.g. "core_test_a1b2c3d4". */ +export function randomCoreSchema(): string { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let suffix = ""; + for (const b of bytes) suffix += SCHEMA_SUFFIX_ALPHABET[b % 36]; + return `core_test_${suffix}`; +} + +// --------------------------------------------------------------------------- +// TestCore — a provisioned, isolated core schema +// --------------------------------------------------------------------------- + +export class TestCore { + readonly schema: string; + private readonly sql: SQL; + + private constructor(sql: SQL, schema: string) { + this.sql = sql; + this.schema = schema; + } + + static async create( + sql: SQL, + options: Omit & { schema?: string } = {}, + ): Promise { + const schema = options.schema ?? randomCoreSchema(); + await migrateCore(sql, { ...options, schema }); + return new TestCore(sql, schema); + } + + async drop(): Promise { + await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); + } +} + +/** + * Provision a fresh core, run `fn` against it, and always drop it afterward. + * Safe to call from concurrent tests — each gets its own unique schema. + */ +export async function withTestCore( + sql: SQL, + options: Omit & { schema?: string }, + fn: (core: TestCore) => Promise, +): Promise { + const core = await TestCore.create(sql, options); + try { + return await fn(core); + } finally { + await core.drop(); + } +} + +/** + * Assert that a query rejects. bun:test's `expect(...).rejects` does not drive + * postgres-js's lazy `PendingQuery` (it only runs when truly awaited), so it + * hangs; awaiting inside try/catch executes the query and observes the error. + */ +export async function expectReject(fn: () => Promise): Promise { + let rejected = false; + try { + await fn(); + } catch { + rejected = true; + } + if (!rejected) { + throw new Error("expected the operation to reject, but it resolved"); + } +} + +// --------------------------------------------------------------------------- +// Schema introspection +// --------------------------------------------------------------------------- + +export async function schemaExists(sql: SQL, name: string): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${name} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function tableExists( + sql: SQL, + schema: string, + table: string, +): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.tables + where table_schema = ${schema} and table_name = ${table} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function listTables(sql: SQL, schema: string): Promise { + const rows = await sql` + select table_name + from information_schema.tables + where table_schema = ${schema} and table_type = 'BASE TABLE' + order by table_name + `; + return rows.map((r) => r.table_name as string); +} + +/** Distinct function names in a schema (overloads collapse to one entry). */ +export async function listFunctions( + sql: SQL, + schema: string, +): Promise { + const rows = await sql` + select distinct p.proname + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = ${schema} + order by p.proname + `; + return rows.map((r) => r.proname as string); +} + +export async function listTriggers( + sql: SQL, + schema: string, + table: string, +): Promise { + const rows = await sql` + select t.tgname + from pg_trigger t + join pg_class c on c.oid = t.tgrelid + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = ${schema} and c.relname = ${table} and not t.tgisinternal + order by t.tgname + `; + return rows.map((r) => r.tgname as string); +} + +/** Whether an extension is installed (in any schema). */ +export async function extensionInstalled( + sql: SQL, + name: string, +): Promise { + const [row] = await sql` + select exists ( + select 1 from pg_extension where extname = ${name} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function appliedMigrations( + sql: SQL, + schema: string, +): Promise { + const rows = await sql.unsafe( + `select name from ${schema}.migration order by name`, + ); + return rows.map((r) => r.name as string); +} + +export async function getSchemaVersion( + sql: SQL, + schema: string, +): Promise { + const [row] = await sql.unsafe(`select version from ${schema}.version`); + return row?.version; +} diff --git a/packages/core/package.json b/packages/core/package.json index a38ca89..0ce4fb9 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "dependencies": { - "@pydantic/logfire-node": "^0.13.1" + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9" } } diff --git a/packages/space/migrate/bootstrap.ts b/packages/space/migrate/bootstrap.ts index 07d67ec..74bc627 100644 --- a/packages/space/migrate/bootstrap.ts +++ b/packages/space/migrate/bootstrap.ts @@ -1,5 +1,6 @@ import { info, reportError, span } from "@pydantic/logfire-node"; -import { type SQL, semver } from "bun"; +import { semver } from "bun"; +import type { ISql, Sql as SQL } from "postgres"; const REQUIRED_EXTENSIONS = [ { name: "citext", minVersion: "1.6" }, @@ -77,13 +78,13 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -async function acquireAdvisoryLock(tx: SQL): Promise { +async function acquireAdvisoryLock(tx: ISql<{}>): Promise { let acquired = false; for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { const [result] = await tx` select pg_try_advisory_xact_lock(${BOOTSTRAP_LOCK_ID}) as acquired `; - if (result.acquired) { + if (result?.acquired) { acquired = true; break; } @@ -97,19 +98,20 @@ async function acquireAdvisoryLock(tx: SQL): Promise { } } -async function ensurePostgresVersion(tx: SQL): Promise { - const [{ server_version_num }] = await tx` +async function ensurePostgresVersion(tx: ISql<{}>): Promise { + const [row] = await tx` select current_setting('server_version_num')::int as server_version_num `; - if (server_version_num < 180000) { + const serverVersionNum = Number(row?.server_version_num); + if (serverVersionNum < 180000) { throw new Error( - `PostgreSQL version 18 or higher is required (found ${server_version_num})`, + `PostgreSQL version 18 or higher is required (found ${serverVersionNum})`, ); } } async function ensureExtension( - tx: SQL, + tx: ISql<{}>, name: string, minVersion: string, ): Promise { diff --git a/packages/space/migrate/migrate.integration.test.ts b/packages/space/migrate/migrate.integration.test.ts new file mode 100644 index 0000000..6dc8a6c --- /dev/null +++ b/packages/space/migrate/migrate.integration.test.ts @@ -0,0 +1,279 @@ +// Integration tests for per-space data-plane migrations (migrateSpace) and the +// shared database bootstrap (bootstrapSpaceDatabase). +// +// Provisioning a space is latency-bound (many sequential statements; ~seconds +// against a remote ghost db), so we provision a small fixed set of spaces once +// in beforeAll — concurrently, via Promise.all — and run fast read-only +// assertions against them. Only the handful of tests that need a private, +// mutable space provision their own. +// +// Tests are serial within the file (Bun 1.3's `test.concurrent` deadlocks when +// many heavy migration transactions overlap). Parallelism comes from two +// places instead: concurrent provisioning in beforeAll, and running the core +// and space suites as separate processes (`bun run test:db`). Spaces are +// isolated by unique `me_` schema, so those processes never collide. +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Sql as SQL } from "postgres"; +import { SPACE_SCHEMA_VERSION } from "../version"; +import { bootstrapSpaceDatabase } from "./bootstrap"; +import { migrateSpace } from "./migrate"; +import { + appliedMigrations, + columnType, + connect, + expectReject, + getIndexReloptions, + getSchemaVersion, + listFunctions, + listIndexes, + listTables, + listTriggers, + schemaExists, + TestSpace, + withTestSpace, +} from "./test-utils"; + +const EXPECTED_TABLES = ["embedding_queue", "memory", "migration", "version"]; + +const EXPECTED_MIGRATIONS = ["001_memory", "002_embedding_queue"]; + +const EXPECTED_MEMORY_FUNCTIONS = [ + "copy_tree", + "count_tree", + "create_memory", + "delete_memory", + "delete_tree", + "get_memory", + "has_tree_access", + "hybrid_search_memory", + "list_tree", + "move_tree", + "patch_memory", + "search_memory", + "tree_access", +]; + +const EXPECTED_QUEUE_FUNCTIONS = [ + "claim_embedding_batch", + "enqueue_embedding", + "prune_embedding_queue", +]; + +const EXPECTED_MEMORY_INDEXES = [ + "memory_content_bm25_idx", + "memory_embedding_hnsw_idx", + "memory_meta_gin_idx", + "memory_temporal_gist_idx", + "memory_tree_gist_idx", +]; + +let sql: SQL; +// Shared spaces, provisioned once. Read-only shape/functional assertions run +// against these; their schemas never change underneath each other. +let canonical: TestSpace; // default params; also used for functional smoke +let dim768: TestSpace; // custom embedding dimension +let customIdx: TestSpace; // custom HNSW + BM25 index parameters + +beforeAll(async () => { + sql = connect(12); + await bootstrapSpaceDatabase(sql); + [canonical, dim768, customIdx] = await Promise.all([ + TestSpace.create(sql), + TestSpace.create(sql, { embeddingDimensions: 768 }), + TestSpace.create(sql, { + hnswM: 8, + hnswEfConstruction: 32, + bm25K1: 1.5, + bm25B: 0.5, + }), + ]); +}); + +afterAll(async () => { + await Promise.all([canonical?.drop(), dim768?.drop(), customIdx?.drop()]); + await sql.end(); +}); + +describe("provisioned space schema", () => { + test("creates the space schema", async () => { + expect(await schemaExists(sql, canonical.schema)).toBe(true); + }); + + test("creates infrastructure and domain tables", async () => { + const tables = await listTables(sql, canonical.schema); + for (const table of EXPECTED_TABLES) { + expect(tables).toContain(table); + } + }); + + test("records every incremental migration exactly once", async () => { + expect(await appliedMigrations(sql, canonical.schema)).toEqual( + EXPECTED_MIGRATIONS, + ); + }); + + test("stamps the schema version", async () => { + expect(await getSchemaVersion(sql, canonical.schema)).toBe( + SPACE_SCHEMA_VERSION, + ); + }); + + test("creates the memory + queue functions", async () => { + const functions = await listFunctions(sql, canonical.schema); + for (const fn of [ + ...EXPECTED_MEMORY_FUNCTIONS, + ...EXPECTED_QUEUE_FUNCTIONS, + ]) { + expect(functions).toContain(fn); + } + }); + + test("creates all memory search indexes", async () => { + const indexes = await listIndexes(sql, canonical.schema, "memory"); + for (const idx of EXPECTED_MEMORY_INDEXES) { + expect(indexes).toContain(idx); + } + }); + + test("memory.embedding defaults to halfvec(1536)", async () => { + expect(await columnType(sql, canonical.schema, "memory", "embedding")).toBe( + "halfvec(1536)", + ); + }); + + test("installs the memory update trigger", async () => { + const triggers = await listTriggers(sql, canonical.schema, "memory"); + expect(triggers).toContain("memory_before_update_trg"); + }); +}); + +describe("migration templating", () => { + test("applies a custom embedding dimension to the table and search fn", async () => { + // The template var drives the column type ... + expect(await columnType(sql, dim768.schema, "memory", "embedding")).toBe( + "halfvec(768)", + ); + // ... and is baked into the search function body's vector casts. + const [row] = await sql.unsafe( + `select pg_get_functiondef(p.oid) as def + from pg_proc p join pg_namespace n on n.oid = p.pronamespace + where n.nspname = '${dim768.schema}' and p.proname = 'search_memory'`, + ); + expect(row?.def).toContain("halfvec(768)"); + }); + + test("applies custom HNSW index parameters", async () => { + const opts = await getIndexReloptions( + sql, + customIdx.schema, + "memory_embedding_hnsw_idx", + ); + expect(opts).toContain("m=8"); + expect(opts).toContain("ef_construction=32"); + }); + + test("applies custom BM25 index parameters", async () => { + const opts = await getIndexReloptions( + sql, + customIdx.schema, + "memory_content_bm25_idx", + ); + expect(opts).toContain("k1=1.5"); + expect(opts).toContain("b=0.5"); + }); +}); + +describe("migration behavior", () => { + test("is idempotent: re-running is safe", async () => { + await withTestSpace(sql, {}, async (space) => { + const before = await appliedMigrations(sql, space.schema); + await migrateSpace(sql, { slug: space.slug }); + expect(await appliedMigrations(sql, space.schema)).toEqual(before); + expect(await getSchemaVersion(sql, space.schema)).toBe( + SPACE_SCHEMA_VERSION, + ); + }); + }); + + test("rejects a downgrade (db version newer than app)", async () => { + await withTestSpace(sql, {}, async (space) => { + await sql.unsafe(`update ${space.schema}.version set version = '99.0.0'`); + await expect(migrateSpace(sql, { slug: space.slug })).rejects.toThrow( + /older than database version/, + ); + }); + }); + + test("rejects invalid slugs before touching the database", async () => { + for (const slug of ["BAD", "short", "way-too-long-slug", "has space12"]) { + await expect(migrateSpace(sql, { slug })).rejects.toThrow( + /Invalid space slug/, + ); + } + }); + + test("provisions distinct spaces independently and in parallel", async () => { + const [a, b] = await Promise.all([ + TestSpace.create(sql), + TestSpace.create(sql), + ]); + try { + expect(a.schema).not.toBe(b.schema); + expect(await schemaExists(sql, a.schema)).toBe(true); + expect(await schemaExists(sql, b.schema)).toBe(true); + // Dropping one leaves the other intact. + await a.drop(); + expect(await schemaExists(sql, a.schema)).toBe(false); + expect(await schemaExists(sql, b.schema)).toBe(true); + } finally { + await a.drop(); + await b.drop(); + } + }); +}); + +describe("provisioned schema is functional", () => { + // Shape assertions read the catalog, so sharing `canonical` with these write + // smoke tests is safe — inserted rows don't affect schema introspection. + test("accepts a memory and fires the update trigger", async () => { + const [row] = await sql.unsafe( + `insert into ${canonical.schema}.memory (content, tree) + values ('hello world', 'a.b') returning id, updated_at`, + ); + expect(row?.id).toBeDefined(); + expect(row?.updated_at).toBeNull(); + + const [updated] = await sql.unsafe( + `update ${canonical.schema}.memory set content = 'changed' + where id = '${row?.id}' returning updated_at`, + ); + expect(updated?.updated_at).not.toBeNull(); + }); + + test("enforces the meta-is-object constraint", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.memory (content, meta) + values ('x', '[]'::jsonb)`, + ), + ); + }); + + test("enforces the temporal range convention", async () => { + // A closed [start,end] range with start < end violates the convention + // (ranges must be inclusive-exclusive); only point-in-time may close upper. + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.memory (content, temporal) + values ('x', '[2024-01-01, 2024-01-02]'::tstzrange)`, + ), + ); + }); +}); + +describe("bootstrapSpaceDatabase", () => { + test("is idempotent", async () => { + await bootstrapSpaceDatabase(sql); + await bootstrapSpaceDatabase(sql); + }); +}); diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts index b5154ba..2bd5bfe 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/space/migrate/migrate.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; -import { SQL, semver } from "bun"; +import { semver } from "bun"; +import type { ISql, Sql as SQL } from "postgres"; import { isValidSlug, slugToSchema } from "../slug"; import { SPACE_SCHEMA_VERSION } from "../version"; import incremental001 from "./incremental/001_memory.sql" with { type: "text" }; @@ -220,7 +221,7 @@ function sleep(ms: number): Promise { } async function acquireAdvisoryLock( - tx: SQL, + tx: ISql<{}>, key1: number, key2: number, ): Promise { @@ -229,7 +230,7 @@ async function acquireAdvisoryLock( const [result] = await tx` select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired `; - if (result.acquired) { + if (result?.acquired) { acquired = true; break; } @@ -240,8 +241,8 @@ async function acquireAdvisoryLock( return acquired; } -async function doesSpaceExist(tx: SQL, schema: string): Promise { - const [{ spaceExists }] = await tx` +async function doesSpaceExist(tx: ISql<{}>, schema: string): Promise { + const [row] = await tx` select exists ( select 1 @@ -249,11 +250,11 @@ async function doesSpaceExist(tx: SQL, schema: string): Promise { where n.nspname = ${schema} ) as "spaceExists" `; - return spaceExists; + return Boolean(row?.spaceExists); } async function provisionSpace( - tx: SQL, + tx: ISql<{}>, schema: string, options: NormalizedMigrateSpaceOptions, ): Promise { @@ -268,7 +269,7 @@ async function provisionSpace( } async function runMigrations( - tx: SQL, + tx: ISql<{}>, schema: string, options: NormalizedMigrateSpaceOptions, ): Promise { @@ -276,9 +277,10 @@ async function runMigrations( await assertSchemaOwnership(tx, schema); // check version - const [{ version: dbVersion }] = await tx` + const [versionRow] = await tx` select version from ${tx(schema)}.version `; + const dbVersion: string = versionRow?.version; const cmp = semver.order(options.schemaVersion, dbVersion); // abort if target is older than the database if (cmp < 0) { @@ -305,7 +307,7 @@ async function runMigrations( ); for (const migration of sorted1) { - const [{ existing }] = await tx` + const [existingRow] = await tx` select exists ( select 1 @@ -314,7 +316,7 @@ async function runMigrations( ) as existing `; - if (existing) { + if (existingRow?.existing) { continue; } @@ -387,7 +389,7 @@ async function runMigrations( } async function executeSqlFile( - tx: SQL, + tx: ISql<{}>, options: NormalizedMigrateSpaceOptions, schema: string, type: string, @@ -431,8 +433,8 @@ function logSqlExecutionError( } function logPostgresSqlLocation(sqlText: string, error: unknown): void { - if (!(error instanceof SQL.PostgresError)) return; - const position = Number(error.position); + // postgres-js sets `position` (1-based) on server errors; non-PG errors won't. + const position = Number((error as { position?: unknown })?.position); if (!Number.isSafeInteger(position) || position < 1) return; const location = sqlLocation(sqlText, position); @@ -479,7 +481,10 @@ function sqlContext(sqlText: string, line: number, column: number): string { return output.join("\n"); } -async function assertSchemaOwnership(tx: SQL, schema: string): Promise { +async function assertSchemaOwnership( + tx: ISql<{}>, + schema: string, +): Promise { const [result] = await tx` select n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner diff --git a/packages/space/migrate/test-utils.ts b/packages/space/migrate/test-utils.ts new file mode 100644 index 0000000..261da28 --- /dev/null +++ b/packages/space/migrate/test-utils.ts @@ -0,0 +1,251 @@ +import type { Sql as SQL } from "postgres"; +import postgres from "postgres"; +import { slugToSchema } from "../slug"; +import { type MigrateSpaceOptions, migrateSpace } from "./migrate"; + +// --------------------------------------------------------------------------- +// Connection +// --------------------------------------------------------------------------- +// +// See packages/core/migrate/test-utils.ts for the connection model. In short: +// provide a PG 18+ database with citext/ltree/vector/pg_textsearch via +// TEST_DATABASE_URL (ghost) or fall back to local docker Postgres. +// +// Spaces isolate cleanly: each space is its own `me_` schema, so many +// can coexist in one database and tests can run concurrently with unique +// slugs — no `create database` needed (ghost forbids it anyway). + +const DEFAULT_TEST_DATABASE_URL = + "postgresql://postgres@127.0.0.1:5432/postgres"; + +export function resolveTestDatabaseUrl(): string { + return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; +} + +export function connect(max = 10): SQL { + // onnotice silences the routine "… already exists, skipping" NOTICEs that the + // idempotent migrations emit (postgres-js prints them to the console by default). + return postgres(resolveTestDatabaseUrl(), { max, onnotice: () => {} }); +} + +const SLUG_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; + +/** A random valid space slug: 12 lowercase alphanumeric chars. */ +export function randomSlug(): string { + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let slug = ""; + for (const b of bytes) slug += SLUG_ALPHABET[b % SLUG_ALPHABET.length]; + return slug; +} + +// --------------------------------------------------------------------------- +// TestSpace — a provisioned, isolated space schema +// --------------------------------------------------------------------------- + +/** + * A migrated space schema in the shared test database. Assumes + * bootstrapSpaceDatabase() has already run (extensions installed) — do that + * once per file in beforeAll. + */ +export class TestSpace { + readonly slug: string; + readonly schema: string; + private readonly sql: SQL; + + private constructor(sql: SQL, slug: string) { + this.sql = sql; + this.slug = slug; + this.schema = slugToSchema(slug); + } + + static async create( + sql: SQL, + options: Omit & { slug?: string } = {}, + ): Promise { + const slug = options.slug ?? randomSlug(); + await migrateSpace(sql, { ...options, slug }); + return new TestSpace(sql, slug); + } + + async drop(): Promise { + await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); + } +} + +/** + * Provision a fresh space, run `fn` against it, and always drop it afterward. + * Safe to call from concurrent tests — each gets its own unique schema. + */ +export async function withTestSpace( + sql: SQL, + options: Omit & { slug?: string }, + fn: (space: TestSpace) => Promise, +): Promise { + const space = await TestSpace.create(sql, options); + try { + return await fn(space); + } finally { + await space.drop(); + } +} + +/** + * Assert that a query rejects. bun:test's `expect(...).rejects` does not drive + * postgres-js's lazy `PendingQuery` (it only runs when truly awaited), so it + * hangs; awaiting inside try/catch executes the query and observes the error. + */ +export async function expectReject(fn: () => Promise): Promise { + let rejected = false; + try { + await fn(); + } catch { + rejected = true; + } + if (!rejected) { + throw new Error("expected the operation to reject, but it resolved"); + } +} + +// --------------------------------------------------------------------------- +// Schema introspection (mirrors packages/core/migrate/test-utils.ts; kept +// local so the package stays self-contained) +// --------------------------------------------------------------------------- + +export async function schemaExists(sql: SQL, name: string): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${name} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function tableExists( + sql: SQL, + schema: string, + table: string, +): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.tables + where table_schema = ${schema} and table_name = ${table} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function listTables(sql: SQL, schema: string): Promise { + const rows = await sql` + select table_name + from information_schema.tables + where table_schema = ${schema} and table_type = 'BASE TABLE' + order by table_name + `; + return rows.map((r) => r.table_name as string); +} + +/** Distinct function names in a schema (overloads collapse to one entry). */ +export async function listFunctions( + sql: SQL, + schema: string, +): Promise { + const rows = await sql` + select distinct p.proname + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = ${schema} + order by p.proname + `; + return rows.map((r) => r.proname as string); +} + +export async function listTriggers( + sql: SQL, + schema: string, + table: string, +): Promise { + const rows = await sql` + select t.tgname + from pg_trigger t + join pg_class c on c.oid = t.tgrelid + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = ${schema} and c.relname = ${table} and not t.tgisinternal + order by t.tgname + `; + return rows.map((r) => r.tgname as string); +} + +/** Fully-resolved type of a column, e.g. "halfvec(768)" or "uuid". */ +export async function columnType( + sql: SQL, + schema: string, + table: string, + column: string, +): Promise { + const [row] = await sql` + select format_type(a.atttypid, a.atttypmod) as type + from pg_attribute a + where a.attrelid = ${`${schema}.${table}`}::regclass + and a.attname = ${column} + and not a.attisdropped + `; + return row?.type ?? null; +} + +export async function listIndexes( + sql: SQL, + schema: string, + table: string, +): Promise { + const rows = await sql` + select indexname from pg_indexes + where schemaname = ${schema} and tablename = ${table} + order by indexname + `; + return rows.map((r) => r.indexname as string); +} + +export async function getIndexDef( + sql: SQL, + schema: string, + index: string, +): Promise { + const [row] = await sql` + select indexdef from pg_indexes + where schemaname = ${schema} and indexname = ${index} + `; + return row?.indexdef ?? null; +} + +/** Storage parameters of an index, e.g. ["m=8", "ef_construction=32"]. */ +export async function getIndexReloptions( + sql: SQL, + schema: string, + index: string, +): Promise { + const [row] = await sql` + select c.reloptions + from pg_class c + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = ${schema} and c.relname = ${index} + `; + return row?.reloptions ?? []; +} + +export async function appliedMigrations( + sql: SQL, + schema: string, +): Promise { + const rows = await sql.unsafe( + `select name from ${schema}.migration order by name`, + ); + return rows.map((r) => r.name as string); +} + +export async function getSchemaVersion( + sql: SQL, + schema: string, +): Promise { + const [row] = await sql.unsafe(`select version from ${schema}.version`); + return row?.version; +} diff --git a/packages/space/package.json b/packages/space/package.json index 92f1a9d..7d5322b 100644 --- a/packages/space/package.json +++ b/packages/space/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "dependencies": { - "@pydantic/logfire-node": "^0.13.1" + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9" } } diff --git a/packages/space/slug.test.ts b/packages/space/slug.test.ts new file mode 100644 index 0000000..bc31008 --- /dev/null +++ b/packages/space/slug.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from "bun:test"; +import { randomSlug } from "./migrate/test-utils"; +import { + isValidSlug, + isValidSpaceSchema, + schemaToSlug, + slugToSchema, +} from "./slug"; + +describe("isValidSlug", () => { + test("accepts 12 lowercase alphanumeric chars", () => { + expect(isValidSlug("abcdef012345")).toBe(true); + expect(isValidSlug("000000000000")).toBe(true); + expect(isValidSlug("zzzzzzzzzzzz")).toBe(true); + }); + + test("rejects wrong length", () => { + expect(isValidSlug("abc")).toBe(false); + expect(isValidSlug("abcdef01234")).toBe(false); // 11 + expect(isValidSlug("abcdef0123456")).toBe(false); // 13 + expect(isValidSlug("")).toBe(false); + }); + + test("rejects uppercase and special characters", () => { + expect(isValidSlug("ABCDEF012345")).toBe(false); + expect(isValidSlug("abcdef-12345")).toBe(false); + expect(isValidSlug("abcdef_12345")).toBe(false); + expect(isValidSlug("abcde 012345")).toBe(false); + }); +}); + +describe("isValidSpaceSchema", () => { + test("accepts me_ prefix + valid slug", () => { + expect(isValidSpaceSchema("me_abcdef012345")).toBe(true); + }); + + test("rejects missing prefix or wrong shape", () => { + expect(isValidSpaceSchema("abcdef012345")).toBe(false); + expect(isValidSpaceSchema("me_ABCDEF012345")).toBe(false); + expect(isValidSpaceSchema("me_abc")).toBe(false); + expect(isValidSpaceSchema("core")).toBe(false); + expect(isValidSpaceSchema("xx_abcdef012345")).toBe(false); + }); +}); + +describe("slugToSchema / schemaToSlug", () => { + test("slugToSchema prepends me_", () => { + expect(slugToSchema("abcdef012345")).toBe("me_abcdef012345"); + }); + + test("schemaToSlug strips the me_ prefix", () => { + expect(schemaToSlug("me_abcdef012345")).toBe("abcdef012345"); + }); + + test("round-trips", () => { + const slug = "0a1b2c3d4e5f"; + expect(schemaToSlug(slugToSchema(slug))).toBe(slug); + expect(isValidSpaceSchema(slugToSchema(slug))).toBe(true); + }); +}); + +describe("randomSlug", () => { + test("always produces a valid, schema-safe slug", () => { + for (let i = 0; i < 1000; i++) { + const slug = randomSlug(); + expect(isValidSlug(slug)).toBe(true); + expect(isValidSpaceSchema(slugToSchema(slug))).toBe(true); + } + }); + + test("is effectively unique across many draws", () => { + const seen = new Set(); + for (let i = 0; i < 10_000; i++) seen.add(randomSlug()); + expect(seen.size).toBe(10_000); + }); +}); diff --git a/scripts/migrate-db.ts b/scripts/migrate-db.ts index 471deca..412d5eb 100644 --- a/scripts/migrate-db.ts +++ b/scripts/migrate-db.ts @@ -6,7 +6,7 @@ import { SPACE_SCHEMA_VERSION, slugToSchema, } from "@memory.build/space"; -import { SQL } from "bun"; +import postgres from "postgres"; const DEFAULT_DATABASE_URL = "postgresql://postgres@127.0.0.1:5432/postgres"; const DEFAULT_SPACE_SLUG = "dev000000001"; @@ -59,7 +59,7 @@ async function main(): Promise { const mode = parseMode(process.argv[2]); const url = databaseUrl(); const spaceSlug = process.env.SPACE_SLUG ?? DEFAULT_SPACE_SLUG; - const sql = new SQL(url); + const sql = postgres(url, { onnotice: () => {} }); console.log(`Database: ${url}`); console.log(`Mode: ${mode}`); @@ -84,7 +84,7 @@ async function main(): Promise { console.log(`Migrated space ${slugToSchema(spaceSlug)}.`); } } finally { - await sql.close(); + await sql.end(); } } diff --git a/scripts/package.json b/scripts/package.json index 8919499..87c9915 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -6,6 +6,7 @@ "@memory.build/core": "workspace:*", "@memory.build/embedding": "workspace:*", "@memory.build/space": "workspace:*", + "postgres": "^3.4.9", "yaml": "^2.7.0" } } From ae0d56bcb48d09e8a6601a332d28127bc687bdf7 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 1 Jun 2026 21:27:59 +0200 Subject: [PATCH 020/156] test: cover agent_tree_access owner-clamp dedup (the max) Real-DB regression for the max(access) + group by tree_path at the end of agent_tree_access. An agent with a broad `foo` grant plus a redundant `foo.bar` grant, clamped against an owner that grants only `foo.bar`, makes both arms of the inner union emit `foo.bar` at different access levels; the max collapses them to the single effective row. Without it the function returns `foo.bar` twice (verified the assertion fails in that case). Also document the --timeout 30000 needed to run a single integration file directly against ghost (bun's 5s default overruns the migrating beforeAll and surfaces as a misleading hook-timeout). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 10 +++ .../core/migrate/migrate.integration.test.ts | 74 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index f5e0459..41118b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,6 +83,16 @@ TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db `testing_me` is the dedicated ghost database for these tests. +To run a single integration file directly, pass `--timeout 30000` (as `test:db` +does). bun's default 5s timeout isn't enough over a remote ghost connection — +the migrating `beforeAll` provisions a full core/space and overruns it, which +surfaces as a misleading "beforeEach/afterEach hook timed out": + +```bash +TEST_DATABASE_URL="$(ghost connect testing_me)" \ + ./bun test --timeout 30000 packages/core/migrate/migrate.integration.test.ts +``` + Isolation is **schema-level** (ghost forbids `create database`): each test provisions its own schema — `core_test_` for core, `me_` for spaces — so the suites are fully concurrent and parallel-safe across files. diff --git a/packages/core/migrate/migrate.integration.test.ts b/packages/core/migrate/migrate.integration.test.ts index 83e5c03..7b69775 100644 --- a/packages/core/migrate/migrate.integration.test.ts +++ b/packages/core/migrate/migrate.integration.test.ts @@ -59,6 +59,15 @@ const EXPECTED_FUNCTIONS = [ const REQUIRED_EXTENSIONS = ["citext", "ltree", "vector", "pg_textsearch"]; +/** A valid space slug: 12 lowercase alphanumerics (see space.slug check). */ +function randomSlug(): string { + const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let slug = ""; + for (const b of bytes) slug += alphabet[b % 36]; + return slug; +} + let sql: SQL; // One migrated core shared by all read-only shape/function assertions. let canonical: TestCore; @@ -204,6 +213,71 @@ describe("access-control functions are callable", () => { }); }); +describe("agent_tree_access clamps agent access to its owner", () => { + // Regression for the `max(x.access)` + `group by tree_path` at the end of + // agent_tree_access (idempotent/003_tree_access.sql). Setup: + // + // agent grants: foo = owner(3), foo.bar = read(1) <- foo.bar is redundant, + // owner grants: foo.bar = write(2) already covered by foo=3 + // + // The inner UNION then emits foo.bar twice with different access levels: + // * arm 1 keeps the agent's (foo.bar, read) — the owner's foo.bar covers it + // * arm 2 keeps the owner's (foo.bar, write) — the agent's foo covers it + // Without the trailing max/group-by, agent_tree_access would return foo.bar + // twice; the effective access is the highest surviving row, (foo.bar, write). + // `foo` itself never surfaces — the owner grants nothing at or above it. + test("collapses the two clamp directions into one row per path", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + + const [space] = await sql.unsafe( + `insert into ${s}.space (slug, name) values ($1, $2) returning id`, + [randomSlug(), "clamp"], + ); + const spaceId = space?.id as string; + + const [owner] = await sql.unsafe( + `insert into ${s}.principal (kind, name) values ('u', 'owner') returning id`, + ); + const ownerId = owner?.id as string; + + const [agent] = await sql.unsafe( + `insert into ${s}.principal (kind, name, owner_id) values ('a', 'agent', $1) returning id`, + [ownerId], + ); + const agentId = agent?.id as string; + + // both principals must belong to the space for the access functions to see them + await sql.unsafe( + `insert into ${s}.principal_space (space_id, principal_id) values ($1, $2), ($1, $3)`, + [spaceId, ownerId, agentId], + ); + + await sql.unsafe( + `insert into ${s}.tree_access (space_id, principal_id, tree_path, access) values + ($1, $2, 'foo', 3), + ($1, $2, 'foo.bar', 1), + ($1, $3, 'foo.bar', 2)`, + [spaceId, agentId, ownerId], + ); + + const rows = await sql.unsafe( + `select tree_path::text as tree_path, access + from ${s}.agent_tree_access($1, $2) + order by tree_path`, + [agentId, spaceId], + ); + const result = rows.map((r) => ({ + tree_path: r.tree_path as string, + access: r.access as number, + })); + + // One clamped row, access collapsed to the max of the two union arms. + expect(result).toEqual([{ tree_path: "foo.bar", access: 2 }]); + }); + }); +}); + describe("migration behavior", () => { test("is idempotent: re-running changes no migration rows or version", async () => { await withTestCore(sql, {}, async (core) => { From 916597479159956f26d2c627c4b9540289df0c51 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 1 Jun 2026 22:01:36 +0200 Subject: [PATCH 021/156] test: auto-reclaim orphaned integration-test schemas before test:db MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hard-interrupted integration runs (SIGKILL, OOM, a timed-out beforeAll) can leave throwaway schemas behind. Add scripts/clean-test-schemas.ts and run it as a pre-step in `test:db`. Safety is by construction — the sweeper only matches names impossible in production: `core_test_*` (prod control plane is the bare `core`) and `metest_*`. To make the latter safe, test spaces now provision under a `metest_` prefix instead of the production `me_`, via a new optional `schema` override on migrateSpace (mirrors migrateCore; defaults to slugToSchema(slug), so production is unchanged). `metest_` also avoids the `me_` engine-schema prefix. Pointed at a real database the sweeper is a no-op. It is age-gated (drops only schemas older than 60 min) so a concurrent `test:db` sharing the database is safe; `test:db:clean:all` forces a full reset. Verified on ghost: drops stale core_test_/metest_, skips fresh ones, and never touches a production-shaped me_ even with --all. Update the two space tests that re-invoke migrateSpace on an existing space to pass the schema override (as the core tests already do), so they target the test schema rather than provisioning a stray me_. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 16 +- package.json | 4 +- .../space/migrate/migrate.integration.test.ts | 8 +- packages/space/migrate/migrate.ts | 21 +- packages/space/migrate/test-utils.ts | 25 ++- scripts/clean-test-schemas.ts | 192 ++++++++++++++++++ 6 files changed, 253 insertions(+), 13 deletions(-) create mode 100644 scripts/clean-test-schemas.ts diff --git a/CLAUDE.md b/CLAUDE.md index 41118b5..7a60ded 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,10 +94,20 @@ TEST_DATABASE_URL="$(ghost connect testing_me)" \ ``` Isolation is **schema-level** (ghost forbids `create database`): each test -provisions its own schema — `core_test_` for core, `me_` for +provisions its own schema — `core_test_` for core, `metest_` for spaces — so the suites are fully concurrent and parallel-safe across files. -The core migrations are templated so production uses `core` while tests target -throwaway schemas and never touch a real control plane. +Both core and space migrations are templated, so production uses `core` / +`me_` while tests target throwaway schemas and never touch real data. +Test spaces deliberately use the `metest_` prefix (not the production `me_`) +via `migrateSpace`'s `schema` override, so leftovers are distinguishable from +real spaces by name alone. + +`test:db` first runs `test:db:clean` (`scripts/clean-test-schemas.ts`) to +reclaim orphaned `core_test_*` / `metest_*` schemas left by hard-interrupted +runs. It is age-gated (only drops schemas older than 60 min, so a concurrent +`test:db` sharing the database is safe) and a no-op against a production +database — neither pattern can match a real schema. Use `test:db:clean:all` +for a deliberate full reset when nothing else is using the database. ## Style Guides diff --git a/package.json b/package.json index 2741833..62d5fbc 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,9 @@ "server": "./bun run packages/server/index.ts", "setup": "./bun scripts/setup.ts", "test": "./bun test packages", - "test:db": "find packages/core packages/space -name '*.integration.test.ts' -print0 | xargs -0 -P 4 -n 1 ./bun test --timeout 30000", + "test:db": "./bun run test:db:clean && find packages/core packages/space -name '*.integration.test.ts' -print0 | xargs -0 -P 4 -n 1 ./bun test --timeout 30000", + "test:db:clean": "./bun scripts/clean-test-schemas.ts", + "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", "typecheck": "tsc --noEmit" }, "devDependencies": { diff --git a/packages/space/migrate/migrate.integration.test.ts b/packages/space/migrate/migrate.integration.test.ts index 6dc8a6c..c46594a 100644 --- a/packages/space/migrate/migrate.integration.test.ts +++ b/packages/space/migrate/migrate.integration.test.ts @@ -187,7 +187,7 @@ describe("migration behavior", () => { test("is idempotent: re-running is safe", async () => { await withTestSpace(sql, {}, async (space) => { const before = await appliedMigrations(sql, space.schema); - await migrateSpace(sql, { slug: space.slug }); + await migrateSpace(sql, { slug: space.slug, schema: space.schema }); expect(await appliedMigrations(sql, space.schema)).toEqual(before); expect(await getSchemaVersion(sql, space.schema)).toBe( SPACE_SCHEMA_VERSION, @@ -198,9 +198,9 @@ describe("migration behavior", () => { test("rejects a downgrade (db version newer than app)", async () => { await withTestSpace(sql, {}, async (space) => { await sql.unsafe(`update ${space.schema}.version set version = '99.0.0'`); - await expect(migrateSpace(sql, { slug: space.slug })).rejects.toThrow( - /older than database version/, - ); + await expect( + migrateSpace(sql, { slug: space.slug, schema: space.schema }), + ).rejects.toThrow(/older than database version/); }); }); diff --git a/packages/space/migrate/migrate.ts b/packages/space/migrate/migrate.ts index 2bd5bfe..f4007fc 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/space/migrate/migrate.ts @@ -53,6 +53,14 @@ const idempotents: Idempotent[] = [ export interface MigrateSpaceOptions { slug: string; + /** + * Override the target schema name. Defaults to `slugToSchema(slug)` (the + * production `me_`). Provided for tests, which provision under a + * `metest_` prefix so leftovers are trivially distinguishable from real + * spaces (see scripts/clean-test-schemas.ts). Must be a valid lowercase SQL + * identifier; the slug is still validated and used for locking/telemetry. + */ + schema?: string; logSqlFiles?: boolean; shardId?: number; embeddingDimensions?: number; @@ -69,6 +77,7 @@ export interface MigrateSpaceOptions { interface NormalizedMigrateSpaceOptions { slug: string; + schema?: string; logSqlFiles: boolean; schemaVersion: string; shardId?: number; @@ -100,10 +109,15 @@ export async function migrateSpace( `Invalid space slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, ); } + if (opts.schema !== undefined && !isValidSchemaName(opts.schema)) { + throw new Error( + `Invalid schema override: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, + ); + } if (!semver.satisfies(opts.schemaVersion, "*")) { throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); } - const schema = slugToSchema(opts.slug); + const schema = opts.schema ?? slugToSchema(opts.slug); const schemaAttributes = { ...attributes, "db.schema": schema }; const [key1, key2] = advisoryLockKey(`memory-space:schema:${schema}`); @@ -175,6 +189,7 @@ function normalizeMigrateSpaceOptions( ): NormalizedMigrateSpaceOptions { return { slug: options.slug, + schema: options.schema, logSqlFiles: options.logSqlFiles ?? false, schemaVersion: SPACE_SCHEMA_VERSION, shardId: options.shardId, @@ -208,6 +223,10 @@ function templateVars( }; } +function isValidSchemaName(schema: string): boolean { + return /^[a-z_][a-z0-9_]*$/.test(schema) && schema.length <= 63; +} + function advisoryLockKey(schema: string): [number, number] { const digest = createHash("sha256").update(schema).digest(); return [digest.readInt32BE(0), digest.readInt32BE(4)]; diff --git a/packages/space/migrate/test-utils.ts b/packages/space/migrate/test-utils.ts index 261da28..77ebed9 100644 --- a/packages/space/migrate/test-utils.ts +++ b/packages/space/migrate/test-utils.ts @@ -1,6 +1,5 @@ import type { Sql as SQL } from "postgres"; import postgres from "postgres"; -import { slugToSchema } from "../slug"; import { type MigrateSpaceOptions, migrateSpace } from "./migrate"; // --------------------------------------------------------------------------- @@ -28,6 +27,20 @@ export function connect(max = 10): SQL { return postgres(resolveTestDatabaseUrl(), { max, onnotice: () => {} }); } +/** + * Test spaces provision under a `metest_` schema instead of the production + * `me_`, so leftover test schemas are distinguishable from real spaces by + * name alone: scripts/clean-test-schemas.ts sweeps `metest_*` (and + * `core_test_*`) and can never touch a production `me_*` space. `metest_` + * deliberately does not start with the `me_` engine-schema prefix. + */ +const TEST_SCHEMA_PREFIX = "metest_"; + +/** The throwaway schema name a test space provisions under (`metest_`). */ +export function testSchema(slug: string): string { + return `${TEST_SCHEMA_PREFIX}${slug}`; +} + const SLUG_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; /** A random valid space slug: 12 lowercase alphanumeric chars. */ @@ -55,15 +68,19 @@ export class TestSpace { private constructor(sql: SQL, slug: string) { this.sql = sql; this.slug = slug; - this.schema = slugToSchema(slug); + this.schema = testSchema(slug); } static async create( sql: SQL, - options: Omit & { slug?: string } = {}, + options: Omit & { + slug?: string; + } = {}, ): Promise { const slug = options.slug ?? randomSlug(); - await migrateSpace(sql, { ...options, slug }); + // Provision under metest_ so leftovers are name-distinguishable from + // production me_ spaces (see clean-test-schemas.ts). + await migrateSpace(sql, { ...options, slug, schema: testSchema(slug) }); return new TestSpace(sql, slug); } diff --git a/scripts/clean-test-schemas.ts b/scripts/clean-test-schemas.ts new file mode 100644 index 0000000..db5f1ed --- /dev/null +++ b/scripts/clean-test-schemas.ts @@ -0,0 +1,192 @@ +#!/usr/bin/env bun +// +// Reclaim orphaned integration-test schemas from the test database. +// +// Integration tests provision throwaway schemas and drop them in teardown, but +// a hard interruption (SIGKILL, OOM, a timed-out `beforeAll`) can leave one +// behind. This script sweeps those leftovers. It runs automatically before +// `test:db` (see package.json) and can be run by hand. +// +// SAFETY — this script issues `drop schema ... cascade`, so it only ever targets +// schema names that are impossible in production: +// +// * `core_test_*` — core tests; production's control plane is the bare `core`. +// * `metest_*` — space tests; production spaces are `me_`. Tests +// deliberately provision under the `metest_` prefix (see +// packages/space/migrate/test-utils.ts) so they never share +// a name with a real space, and `metest_` does not start +// with the `me_` engine prefix. +// +// The result: pointed at a real database, this script is a no-op — neither +// pattern can match a production schema. +// +// By default it only drops schemas older than --older-than-min (default 60), +// using each schema's `version.at` timestamp. That keeps it safe to run as a +// pre-step even while another `test:db` invocation shares the database: that +// run's freshly-provisioned schemas are younger than the threshold and are left +// alone. Pass --all to ignore age (a deliberate full reset — only do this when +// nothing else is using the database). + +import postgres, { type Sql } from "postgres"; + +const DEFAULT_TEST_DATABASE_URL = + "postgresql://postgres@127.0.0.1:5432/postgres"; + +// Production-impossible test schema patterns. core_test_, metest_. +const TEST_SCHEMA_PATTERNS = ["^core_test_[a-z0-9]+$", "^metest_[a-z0-9]{12}$"]; + +const SAFE_IDENTIFIER = /^[a-z_][a-z0-9_]*$/; + +interface Options { + all: boolean; + olderThanMin: number; + quiet: boolean; +} + +function parseArgs(argv: string[]): Options { + const opts: Options = { all: false, olderThanMin: 60, quiet: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--help" || arg === "-h") { + console.log( + `Usage: ./bun scripts/clean-test-schemas.ts [--all] [--older-than-min N] [--quiet] + +Drops orphaned integration-test schemas from TEST_DATABASE_URL (default +${DEFAULT_TEST_DATABASE_URL}): core_test_* and metest_* schemas. Safe against +production databases (no-op there — neither pattern can match a real schema). + + --all Ignore age; drop every matching schema. Use only when no + other test run shares the database. + --older-than-min N Only drop schemas older than N minutes (default 60). + --quiet Only print when something is dropped or on error.`, + ); + process.exit(0); + } else if (arg === "--all") { + opts.all = true; + } else if (arg === "--quiet") { + opts.quiet = true; + } else if (arg === "--older-than-min") { + const next = argv[++i]; + const n = Number(next); + if (!Number.isFinite(n) || n < 0) { + console.error(`Invalid --older-than-min value: ${next}`); + process.exit(2); + } + opts.olderThanMin = n; + } else { + console.error(`Unknown argument: ${arg}`); + process.exit(2); + } + } + return opts; +} + +function testDatabaseUrl(): string { + return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; +} + +async function findCandidates(sql: Sql): Promise { + // OR the (constant, production-impossible) patterns as bound `~` tests. + const match = TEST_SCHEMA_PATTERNS.map((p) => sql`n.nspname ~ ${p}`).reduce( + (acc, frag) => sql`${acc} or ${frag}`, + ); + const rows = await sql<{ schema: string }[]>` + select n.nspname as schema + from pg_namespace n + where ${match} + order by n.nspname + `; + return rows.map((r) => r.schema); +} + +/** + * Age of a schema in minutes from its singleton `version.at`, or null when the + * schema has no readable version row (a partial/failed provision). In safe mode + * those are skipped — they may belong to a concurrent run still provisioning. + */ +async function schemaAgeMinutes( + sql: Sql, + schema: string, +): Promise { + try { + const rows = await sql.unsafe<{ age_min: number }[]>( + `select extract(epoch from (now() - at)) / 60 as age_min + from ${schema}.version + limit 1`, + ); + const [row] = rows; + return row ? Number(row.age_min) : null; + } catch { + return null; + } +} + +async function main(): Promise { + const opts = parseArgs(process.argv.slice(2)); + const log = (msg: string) => { + if (!opts.quiet) console.error(msg); + }; + + // Short timeouts: never let cleanup hang a test run on a lock. statement_timeout + // and lock_timeout are libpq startup params here, in milliseconds. + const sql = postgres(testDatabaseUrl(), { + max: 1, + connect_timeout: 10, + idle_timeout: 5, + onnotice: () => {}, + connection: { statement_timeout: 10_000, lock_timeout: 3_000 }, + }); + + const dropped: string[] = []; + try { + const candidates = await findCandidates(sql); + if (candidates.length === 0) { + log("[clean-test-schemas] no test schemas found."); + return; + } + + for (const schema of candidates) { + // Defense in depth: the patterns already constrain this, but never + // interpolate a name that isn't a plain lowercase identifier. + if (!SAFE_IDENTIFIER.test(schema) || schema.length > 63) { + log(`[clean-test-schemas] skip ${schema}: unexpected identifier`); + continue; + } + + if (!opts.all) { + const age = await schemaAgeMinutes(sql, schema); + if (age === null) { + log(`[clean-test-schemas] skip ${schema}: no version row (recent?)`); + continue; + } + if (age < opts.olderThanMin) { + log( + `[clean-test-schemas] skip ${schema}: ${age.toFixed(0)}m old < ${opts.olderThanMin}m`, + ); + continue; + } + } + + try { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + dropped.push(schema); + } catch (error) { + log(`[clean-test-schemas] failed to drop ${schema}: ${error}`); + } + } + } catch (error) { + // Best-effort: never block the test run on a cleanup hiccup. If the database + // is genuinely unreachable, the tests themselves will surface it. + console.error(`[clean-test-schemas] skipped (cleanup error): ${error}`); + } finally { + await sql.end(); + } + + if (dropped.length > 0) { + console.error( + `[clean-test-schemas] dropped ${dropped.length} schema(s): ${dropped.join(", ")}`, + ); + } +} + +await main(); From 2d21b82cc5e3309dc649726cb03b3bb1a3ffd5fc Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 2 Jun 2026 15:17:26 +0200 Subject: [PATCH 022/156] refactor: merge core + space into one @memory.build/database package Co-locate the control plane (core schema) and data plane (per-space me_ schemas) in a single @memory.build/database package, kept as separate core/ and space/ modules. The team runs them in one database/deployment and pgdog sharding of spaces is off the table for now; the per-slug schema model and the set-local-pgdog.shard code stay in space/ so re-splitting later is cheap. - packages/{core,space} -> packages/database/{core,space} (git mv, history preserved); single package.json/tsconfig; index.ts re-exports both modules. - Rewire scripts/migrate-db.ts + scripts/package.json + root test:db to @memory.build/database. - ISql<{}> -> ISql (clears the noBannedTypes lint warnings). - TODO.md records the decision; the test-utils/migration-runner consolidations become internal modules. Verified: typecheck + lint clean; 752 unit tests pass; db integration on ghost (core 18, space 19) pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 6 +- TODO.md | 65 +++++++++---------- bun.lock | 19 ++---- package.json | 2 +- packages/{ => database}/core/index.ts | 0 .../core/migrate/idempotent/000_update.sql | 0 .../idempotent/001_principal_space.sql | 0 .../migrate/idempotent/002_group_member.sql | 0 .../migrate/idempotent/003_tree_access.sql | 0 .../core/migrate/idempotent/sql.d.ts | 0 .../core/migrate/incremental/001_space.sql | 0 .../migrate/incremental/002_principal.sql | 0 .../incremental/003_principal_space.sql | 0 .../migrate/incremental/004_group_member.sql | 0 .../migrate/incremental/005_tree_access.sql | 0 .../core/migrate/incremental/006_api_key.sql | 0 .../core/migrate/incremental/sql.d.ts | 0 .../core/migrate/migrate.integration.test.ts | 0 .../{ => database}/core/migrate/migrate.ts | 19 +++--- .../{ => database}/core/migrate/provision.sql | 0 packages/{ => database}/core/migrate/sql.d.ts | 0 .../{ => database}/core/migrate/test-utils.ts | 0 packages/{ => database}/core/version.ts | 0 packages/database/index.ts | 7 ++ packages/{core => database}/package.json | 2 +- packages/{ => database}/space/index.ts | 0 .../{ => database}/space/migrate/bootstrap.ts | 6 +- .../space/migrate/idempotent/001_memory.sql | 0 .../space/migrate/idempotent/002_search.sql | 0 .../idempotent/003_embedding_queue.sql | 0 .../space/migrate/idempotent/sql.d.ts | 0 .../space/migrate/incremental/001_memory.sql | 0 .../incremental/002_embedding_queue.sql | 0 .../space/migrate/incremental/sql.d.ts | 0 .../space/migrate/migrate.integration.test.ts | 0 .../{ => database}/space/migrate/migrate.ts | 15 ++--- .../space/migrate/provision.sql | 0 .../{ => database}/space/migrate/sql.d.ts | 0 .../space/migrate/test-utils.ts | 0 packages/{ => database}/space/slug.test.ts | 0 packages/{ => database}/space/slug.ts | 0 packages/{ => database}/space/version.ts | 0 packages/{core => database}/tsconfig.json | 0 packages/space/package.json | 10 --- packages/space/tsconfig.json | 4 -- scripts/migrate-db.ts | 5 +- scripts/package.json | 3 +- 47 files changed, 66 insertions(+), 97 deletions(-) rename packages/{ => database}/core/index.ts (100%) rename packages/{ => database}/core/migrate/idempotent/000_update.sql (100%) rename packages/{ => database}/core/migrate/idempotent/001_principal_space.sql (100%) rename packages/{ => database}/core/migrate/idempotent/002_group_member.sql (100%) rename packages/{ => database}/core/migrate/idempotent/003_tree_access.sql (100%) rename packages/{ => database}/core/migrate/idempotent/sql.d.ts (100%) rename packages/{ => database}/core/migrate/incremental/001_space.sql (100%) rename packages/{ => database}/core/migrate/incremental/002_principal.sql (100%) rename packages/{ => database}/core/migrate/incremental/003_principal_space.sql (100%) rename packages/{ => database}/core/migrate/incremental/004_group_member.sql (100%) rename packages/{ => database}/core/migrate/incremental/005_tree_access.sql (100%) rename packages/{ => database}/core/migrate/incremental/006_api_key.sql (100%) rename packages/{ => database}/core/migrate/incremental/sql.d.ts (100%) rename packages/{ => database}/core/migrate/migrate.integration.test.ts (100%) rename packages/{ => database}/core/migrate/migrate.ts (98%) rename packages/{ => database}/core/migrate/provision.sql (100%) rename packages/{ => database}/core/migrate/sql.d.ts (100%) rename packages/{ => database}/core/migrate/test-utils.ts (100%) rename packages/{ => database}/core/version.ts (100%) create mode 100644 packages/database/index.ts rename packages/{core => database}/package.json (81%) rename packages/{ => database}/space/index.ts (100%) rename packages/{ => database}/space/migrate/bootstrap.ts (96%) rename packages/{ => database}/space/migrate/idempotent/001_memory.sql (100%) rename packages/{ => database}/space/migrate/idempotent/002_search.sql (100%) rename packages/{ => database}/space/migrate/idempotent/003_embedding_queue.sql (100%) rename packages/{ => database}/space/migrate/idempotent/sql.d.ts (100%) rename packages/{ => database}/space/migrate/incremental/001_memory.sql (100%) rename packages/{ => database}/space/migrate/incremental/002_embedding_queue.sql (100%) rename packages/{ => database}/space/migrate/incremental/sql.d.ts (100%) rename packages/{ => database}/space/migrate/migrate.integration.test.ts (100%) rename packages/{ => database}/space/migrate/migrate.ts (98%) rename packages/{ => database}/space/migrate/provision.sql (100%) rename packages/{ => database}/space/migrate/sql.d.ts (100%) rename packages/{ => database}/space/migrate/test-utils.ts (100%) rename packages/{ => database}/space/slug.test.ts (100%) rename packages/{ => database}/space/slug.ts (100%) rename packages/{ => database}/space/version.ts (100%) rename packages/{core => database}/tsconfig.json (100%) delete mode 100644 packages/space/package.json delete mode 100644 packages/space/tsconfig.json diff --git a/CLAUDE.md b/CLAUDE.md index 7a60ded..6870ea5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,7 +90,7 @@ surfaces as a misleading "beforeEach/afterEach hook timed out": ```bash TEST_DATABASE_URL="$(ghost connect testing_me)" \ - ./bun test --timeout 30000 packages/core/migrate/migrate.integration.test.ts + ./bun test --timeout 30000 packages/database/core/migrate/migrate.integration.test.ts ``` Isolation is **schema-level** (ghost forbids `create database`): each test @@ -143,8 +143,8 @@ pool — e.g. the engine/accounts pools in `packages/server/index.ts` — can we server until restart. Both `postgres` (postgres.js) and `pg` fix it on the Bun runtime; we use **postgres.js** because `Bun.SQL`'s API was modeled on it, so it's a near-drop-in. -**Done & verified (local + ghost):** the migrate path — `packages/core/migrate/*`, -`packages/space/migrate/*` (incl. `test-utils.ts`), and `scripts/migrate-db.ts`. +**Done & verified (local + ghost):** the migrate path — `packages/database/core/migrate/*`, +`packages/database/space/migrate/*` (incl. `test-utils.ts`), and `scripts/migrate-db.ts`. **Remaining**, package by package, each behind its own integration tests: `packages/engine` (`db.ts`, `ops/*`, `migrate/*`), `packages/accounts` (`db.ts`, `ops/*`, diff --git a/TODO.md b/TODO.md index f2a0edd..1f1050c 100644 --- a/TODO.md +++ b/TODO.md @@ -1,47 +1,40 @@ # TODO Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, -see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). The two -consolidations below are best done as part of that rollout, since it touches every -package's migration and test code anyway. +see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). -## Open question: should `core` and `space` be one package? +## Decision: `core` and `space` are one package (`@memory.build/database`) -They're separate packages today (`packages/core`, `packages/space`) with no runtime -dependency between them. The consolidations below keep bumping into that split — -sharing code requires a separate shared/dev package rather than a plain internal -module. So: should they merge into one package? +Resolved (2026-06): merged `packages/core` + `packages/space` into a single +`@memory.build/database`, kept as separate `core/` and `space/` modules. The team +co-locates the control plane and data plane in one database/deployment, and pgdog +sharding/distribution of spaces is off the table for now. The per-slug schema model +and the `set local pgdog.shard` code stay in the `space/` module, so re-splitting +later is cheap if distribution returns. -- **Merge** → sharing the test-utils and migration runner becomes trivial (internal - modules, no extra package); they're conceptually the two halves (control plane + - data plane) of one system. -- **Keep separate** → preserves a clean boundary (space already has zero references to - core) and keeps the door open to deploying/scaling them differently (e.g. space - schemas sharded across many DBs via pgdog, core centralized). +- [ ] Keep `space/` free of `core/` imports (and vice versa) so the re-split escape + hatch stays open — worth a Biome `noRestrictedImports` rule to enforce it. -Decide this first — it determines whether the consolidations below land as internal -modules (merged) or a shared package (separate). +## Consolidate duplicated test-utils (now an internal module) -## Consolidate duplicated test-utils +`packages/database/core/migrate/test-utils.ts` and +`packages/database/space/migrate/test-utils.ts` duplicate ~110 lines of generic, +driver-level helpers: `resolveTestDatabaseUrl`, `connect`, `expectReject`, and schema +introspection (`schemaExists`, `tableExists`, `listTables`, `listFunctions`, +`listTriggers`, `appliedMigrations`, `getSchemaVersion`). -`packages/core/migrate/test-utils.ts` and `packages/space/migrate/test-utils.ts` -duplicate ~110 lines of generic, driver-level helpers: `resolveTestDatabaseUrl`, -`connect`, `expectReject`, and schema introspection (`schemaExists`, `tableExists`, -`listTables`, `listFunctions`, `listTriggers`, `appliedMigrations`, -`getSchemaVersion`). `packages/engine` and `packages/accounts` also carry their own -copies of some of these (~4 repo-wide copies of `tableExists`/`schemaExists`). +- [ ] Move the generic helpers into one shared internal module (e.g. + `packages/database/migrate/test-utils.ts`) imported by both core/ and space/ + tests. Keep package-specific provisioning where it is: `TestCore`/`TestSpace`, + `randomCoreSchema`/`randomSlug`, `columnType`, the index helpers. +- [ ] `engine`/`accounts` carry their own `tableExists`/`schemaExists` copies too; + they're separate packages, so sharing with them still needs a dev-only package + (or fold them in during the postgres.js rollout). -- [ ] Extract the generic helpers into a shared **dev-only** package (e.g. - `@memory.build/db-testkit`) added as a `devDependency` where needed. Keep - package-specific provisioning in each package: `TestCore`/`TestSpace`, - `randomCoreSchema`/`randomSlug`, `columnType`, the index helpers. Test-only, - so it doesn't couple the packages at runtime. -- [ ] Move `engine`/`accounts` test-utils onto it too, removing the older duplicates. +## Consolidate the migration runner logic (now an internal module) -## Consolidate the migration runner logic - -`packages/core/migrate/migrate.ts`, `packages/space/migrate/migrate.ts`, and -`packages/space/migrate/bootstrap.ts` duplicate most of the migration machinery: +`packages/database/core/migrate/migrate.ts`, `.../space/migrate/migrate.ts`, and +`.../space/migrate/bootstrap.ts` duplicate most of the migration machinery: - advisory locking (`advisoryLockKey`, `acquireAdvisoryLock`, retry/backoff) - SQL-file execution with error-location logging (`executeSqlFile`, @@ -52,7 +45,7 @@ copies of some of these (~4 repo-wide copies of `tableExists`/`schemaExists`). - the incremental-once / idempotent-always runner, with version + migration tracking - telemetry span wrapping -- [ ] Extract this into a shared migration util (e.g. `@memory.build/migrate-kit`) +- [ ] Extract into a shared internal module (e.g. `packages/database/migrate/_kit.ts`) parameterized by schema name, the ordered incremental/idempotent SQL lists, and - template vars. `migrateCore` / `migrateSpace` / `bootstrapSpaceDatabase` then - become thin callers, leaving each `migrate.ts` with only its schema-specific bits. + template vars. `migrateCore` / `migrateSpace` / `bootstrapSpaceDatabase` become + thin callers, leaving each `migrate.ts` with only its schema-specific bits. diff --git a/bun.lock b/bun.lock index ec47cfe..728b6e7 100644 --- a/bun.lock +++ b/bun.lock @@ -49,8 +49,8 @@ "typescript", ], }, - "packages/core": { - "name": "@memory.build/core", + "packages/database": { + "name": "@memory.build/database", "version": "0.2.5", "dependencies": { "@pydantic/logfire-node": "^0.13.1", @@ -131,14 +131,6 @@ "zod": "^4.0.0", }, }, - "packages/space": { - "name": "@memory.build/space", - "version": "0.2.5", - "dependencies": { - "@pydantic/logfire-node": "^0.13.1", - "postgres": "^3.4.9", - }, - }, "packages/web": { "name": "@memory.build/web", "version": "0.1.17", @@ -179,9 +171,8 @@ "scripts": { "name": "scripts", "dependencies": { - "@memory.build/core": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", - "@memory.build/space": "workspace:*", "postgres": "^3.4.9", "yaml": "^2.7.0", }, @@ -388,7 +379,7 @@ "@memory.build/client": ["@memory.build/client@workspace:packages/client"], - "@memory.build/core": ["@memory.build/core@workspace:packages/core"], + "@memory.build/database": ["@memory.build/database@workspace:packages/database"], "@memory.build/docs-site": ["@memory.build/docs-site@workspace:packages/docs-site"], @@ -398,8 +389,6 @@ "@memory.build/protocol": ["@memory.build/protocol@workspace:packages/protocol"], - "@memory.build/space": ["@memory.build/space@workspace:packages/space"], - "@memory.build/web": ["@memory.build/web@workspace:packages/web"], "@memory.build/worker": ["@memory.build/worker@workspace:packages/worker"], diff --git a/package.json b/package.json index 62d5fbc..78b32ba 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "server": "./bun run packages/server/index.ts", "setup": "./bun scripts/setup.ts", "test": "./bun test packages", - "test:db": "./bun run test:db:clean && find packages/core packages/space -name '*.integration.test.ts' -print0 | xargs -0 -P 4 -n 1 ./bun test --timeout 30000", + "test:db": "./bun run test:db:clean && find packages/database -name '*.integration.test.ts' -print0 | xargs -0 -P 4 -n 1 ./bun test --timeout 30000", "test:db:clean": "./bun scripts/clean-test-schemas.ts", "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", "typecheck": "tsc --noEmit" diff --git a/packages/core/index.ts b/packages/database/core/index.ts similarity index 100% rename from packages/core/index.ts rename to packages/database/core/index.ts diff --git a/packages/core/migrate/idempotent/000_update.sql b/packages/database/core/migrate/idempotent/000_update.sql similarity index 100% rename from packages/core/migrate/idempotent/000_update.sql rename to packages/database/core/migrate/idempotent/000_update.sql diff --git a/packages/core/migrate/idempotent/001_principal_space.sql b/packages/database/core/migrate/idempotent/001_principal_space.sql similarity index 100% rename from packages/core/migrate/idempotent/001_principal_space.sql rename to packages/database/core/migrate/idempotent/001_principal_space.sql diff --git a/packages/core/migrate/idempotent/002_group_member.sql b/packages/database/core/migrate/idempotent/002_group_member.sql similarity index 100% rename from packages/core/migrate/idempotent/002_group_member.sql rename to packages/database/core/migrate/idempotent/002_group_member.sql diff --git a/packages/core/migrate/idempotent/003_tree_access.sql b/packages/database/core/migrate/idempotent/003_tree_access.sql similarity index 100% rename from packages/core/migrate/idempotent/003_tree_access.sql rename to packages/database/core/migrate/idempotent/003_tree_access.sql diff --git a/packages/core/migrate/idempotent/sql.d.ts b/packages/database/core/migrate/idempotent/sql.d.ts similarity index 100% rename from packages/core/migrate/idempotent/sql.d.ts rename to packages/database/core/migrate/idempotent/sql.d.ts diff --git a/packages/core/migrate/incremental/001_space.sql b/packages/database/core/migrate/incremental/001_space.sql similarity index 100% rename from packages/core/migrate/incremental/001_space.sql rename to packages/database/core/migrate/incremental/001_space.sql diff --git a/packages/core/migrate/incremental/002_principal.sql b/packages/database/core/migrate/incremental/002_principal.sql similarity index 100% rename from packages/core/migrate/incremental/002_principal.sql rename to packages/database/core/migrate/incremental/002_principal.sql diff --git a/packages/core/migrate/incremental/003_principal_space.sql b/packages/database/core/migrate/incremental/003_principal_space.sql similarity index 100% rename from packages/core/migrate/incremental/003_principal_space.sql rename to packages/database/core/migrate/incremental/003_principal_space.sql diff --git a/packages/core/migrate/incremental/004_group_member.sql b/packages/database/core/migrate/incremental/004_group_member.sql similarity index 100% rename from packages/core/migrate/incremental/004_group_member.sql rename to packages/database/core/migrate/incremental/004_group_member.sql diff --git a/packages/core/migrate/incremental/005_tree_access.sql b/packages/database/core/migrate/incremental/005_tree_access.sql similarity index 100% rename from packages/core/migrate/incremental/005_tree_access.sql rename to packages/database/core/migrate/incremental/005_tree_access.sql diff --git a/packages/core/migrate/incremental/006_api_key.sql b/packages/database/core/migrate/incremental/006_api_key.sql similarity index 100% rename from packages/core/migrate/incremental/006_api_key.sql rename to packages/database/core/migrate/incremental/006_api_key.sql diff --git a/packages/core/migrate/incremental/sql.d.ts b/packages/database/core/migrate/incremental/sql.d.ts similarity index 100% rename from packages/core/migrate/incremental/sql.d.ts rename to packages/database/core/migrate/incremental/sql.d.ts diff --git a/packages/core/migrate/migrate.integration.test.ts b/packages/database/core/migrate/migrate.integration.test.ts similarity index 100% rename from packages/core/migrate/migrate.integration.test.ts rename to packages/database/core/migrate/migrate.integration.test.ts diff --git a/packages/core/migrate/migrate.ts b/packages/database/core/migrate/migrate.ts similarity index 98% rename from packages/core/migrate/migrate.ts rename to packages/database/core/migrate/migrate.ts index 204d130..198409b 100644 --- a/packages/core/migrate/migrate.ts +++ b/packages/database/core/migrate/migrate.ts @@ -248,7 +248,7 @@ function sleep(ms: number): Promise { } async function acquireAdvisoryLock( - tx: ISql<{}>, + tx: ISql, key1: number, key2: number, ): Promise { @@ -268,7 +268,7 @@ async function acquireAdvisoryLock( return acquired; } -async function doesCoreExist(tx: ISql<{}>, schema: string): Promise { +async function doesCoreExist(tx: ISql, schema: string): Promise { const [row] = await tx` select exists ( @@ -281,7 +281,7 @@ async function doesCoreExist(tx: ISql<{}>, schema: string): Promise { } async function provisionCore( - tx: ISql<{}>, + tx: ISql, options: NormalizedMigrateCoreOptions, ): Promise { await executeSqlFile( @@ -293,7 +293,7 @@ async function provisionCore( ); } -async function ensurePostgresVersion(tx: ISql<{}>): Promise { +async function ensurePostgresVersion(tx: ISql): Promise { const [row] = await tx` select current_setting('server_version_num')::int as server_version_num `; @@ -306,7 +306,7 @@ async function ensurePostgresVersion(tx: ISql<{}>): Promise { } async function ensureExtension( - tx: ISql<{}>, + tx: ISql, name: string, minVersion: string, ): Promise { @@ -348,7 +348,7 @@ async function ensureExtension( } async function runMigrations( - tx: ISql<{}>, + tx: ISql, options: NormalizedMigrateCoreOptions, ): Promise { const schema = options.schema; @@ -450,7 +450,7 @@ async function runMigrations( } async function executeSqlFile( - tx: ISql<{}>, + tx: ISql, options: NormalizedMigrateCoreOptions, type: string, file: string, @@ -537,10 +537,7 @@ function sqlContext(sqlText: string, line: number, column: number): string { return output.join("\n"); } -async function assertSchemaOwnership( - tx: ISql<{}>, - schema: string, -): Promise { +async function assertSchemaOwnership(tx: ISql, schema: string): Promise { const [result] = await tx` select n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner diff --git a/packages/core/migrate/provision.sql b/packages/database/core/migrate/provision.sql similarity index 100% rename from packages/core/migrate/provision.sql rename to packages/database/core/migrate/provision.sql diff --git a/packages/core/migrate/sql.d.ts b/packages/database/core/migrate/sql.d.ts similarity index 100% rename from packages/core/migrate/sql.d.ts rename to packages/database/core/migrate/sql.d.ts diff --git a/packages/core/migrate/test-utils.ts b/packages/database/core/migrate/test-utils.ts similarity index 100% rename from packages/core/migrate/test-utils.ts rename to packages/database/core/migrate/test-utils.ts diff --git a/packages/core/version.ts b/packages/database/core/version.ts similarity index 100% rename from packages/core/version.ts rename to packages/database/core/version.ts diff --git a/packages/database/index.ts b/packages/database/index.ts new file mode 100644 index 0000000..7b9a573 --- /dev/null +++ b/packages/database/index.ts @@ -0,0 +1,7 @@ +// Control plane (`core` schema) and data plane (per-space `me_` schemas) +// live in one package; they are co-located in a single database/deployment. +// Kept as separate `core/` and `space/` modules so the boundary stays clean +// (space must not import core) and the split is easy to undo if spaces are ever +// distributed across databases again. +export * from "./core"; +export * from "./space"; diff --git a/packages/core/package.json b/packages/database/package.json similarity index 81% rename from packages/core/package.json rename to packages/database/package.json index 0ce4fb9..7953e99 100644 --- a/packages/core/package.json +++ b/packages/database/package.json @@ -1,5 +1,5 @@ { - "name": "@memory.build/core", + "name": "@memory.build/database", "version": "0.2.5", "private": true, "type": "module", diff --git a/packages/space/index.ts b/packages/database/space/index.ts similarity index 100% rename from packages/space/index.ts rename to packages/database/space/index.ts diff --git a/packages/space/migrate/bootstrap.ts b/packages/database/space/migrate/bootstrap.ts similarity index 96% rename from packages/space/migrate/bootstrap.ts rename to packages/database/space/migrate/bootstrap.ts index 74bc627..6a52e71 100644 --- a/packages/space/migrate/bootstrap.ts +++ b/packages/database/space/migrate/bootstrap.ts @@ -78,7 +78,7 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -async function acquireAdvisoryLock(tx: ISql<{}>): Promise { +async function acquireAdvisoryLock(tx: ISql): Promise { let acquired = false; for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { const [result] = await tx` @@ -98,7 +98,7 @@ async function acquireAdvisoryLock(tx: ISql<{}>): Promise { } } -async function ensurePostgresVersion(tx: ISql<{}>): Promise { +async function ensurePostgresVersion(tx: ISql): Promise { const [row] = await tx` select current_setting('server_version_num')::int as server_version_num `; @@ -111,7 +111,7 @@ async function ensurePostgresVersion(tx: ISql<{}>): Promise { } async function ensureExtension( - tx: ISql<{}>, + tx: ISql, name: string, minVersion: string, ): Promise { diff --git a/packages/space/migrate/idempotent/001_memory.sql b/packages/database/space/migrate/idempotent/001_memory.sql similarity index 100% rename from packages/space/migrate/idempotent/001_memory.sql rename to packages/database/space/migrate/idempotent/001_memory.sql diff --git a/packages/space/migrate/idempotent/002_search.sql b/packages/database/space/migrate/idempotent/002_search.sql similarity index 100% rename from packages/space/migrate/idempotent/002_search.sql rename to packages/database/space/migrate/idempotent/002_search.sql diff --git a/packages/space/migrate/idempotent/003_embedding_queue.sql b/packages/database/space/migrate/idempotent/003_embedding_queue.sql similarity index 100% rename from packages/space/migrate/idempotent/003_embedding_queue.sql rename to packages/database/space/migrate/idempotent/003_embedding_queue.sql diff --git a/packages/space/migrate/idempotent/sql.d.ts b/packages/database/space/migrate/idempotent/sql.d.ts similarity index 100% rename from packages/space/migrate/idempotent/sql.d.ts rename to packages/database/space/migrate/idempotent/sql.d.ts diff --git a/packages/space/migrate/incremental/001_memory.sql b/packages/database/space/migrate/incremental/001_memory.sql similarity index 100% rename from packages/space/migrate/incremental/001_memory.sql rename to packages/database/space/migrate/incremental/001_memory.sql diff --git a/packages/space/migrate/incremental/002_embedding_queue.sql b/packages/database/space/migrate/incremental/002_embedding_queue.sql similarity index 100% rename from packages/space/migrate/incremental/002_embedding_queue.sql rename to packages/database/space/migrate/incremental/002_embedding_queue.sql diff --git a/packages/space/migrate/incremental/sql.d.ts b/packages/database/space/migrate/incremental/sql.d.ts similarity index 100% rename from packages/space/migrate/incremental/sql.d.ts rename to packages/database/space/migrate/incremental/sql.d.ts diff --git a/packages/space/migrate/migrate.integration.test.ts b/packages/database/space/migrate/migrate.integration.test.ts similarity index 100% rename from packages/space/migrate/migrate.integration.test.ts rename to packages/database/space/migrate/migrate.integration.test.ts diff --git a/packages/space/migrate/migrate.ts b/packages/database/space/migrate/migrate.ts similarity index 98% rename from packages/space/migrate/migrate.ts rename to packages/database/space/migrate/migrate.ts index f4007fc..fb5beee 100644 --- a/packages/space/migrate/migrate.ts +++ b/packages/database/space/migrate/migrate.ts @@ -240,7 +240,7 @@ function sleep(ms: number): Promise { } async function acquireAdvisoryLock( - tx: ISql<{}>, + tx: ISql, key1: number, key2: number, ): Promise { @@ -260,7 +260,7 @@ async function acquireAdvisoryLock( return acquired; } -async function doesSpaceExist(tx: ISql<{}>, schema: string): Promise { +async function doesSpaceExist(tx: ISql, schema: string): Promise { const [row] = await tx` select exists ( @@ -273,7 +273,7 @@ async function doesSpaceExist(tx: ISql<{}>, schema: string): Promise { } async function provisionSpace( - tx: ISql<{}>, + tx: ISql, schema: string, options: NormalizedMigrateSpaceOptions, ): Promise { @@ -288,7 +288,7 @@ async function provisionSpace( } async function runMigrations( - tx: ISql<{}>, + tx: ISql, schema: string, options: NormalizedMigrateSpaceOptions, ): Promise { @@ -408,7 +408,7 @@ async function runMigrations( } async function executeSqlFile( - tx: ISql<{}>, + tx: ISql, options: NormalizedMigrateSpaceOptions, schema: string, type: string, @@ -500,10 +500,7 @@ function sqlContext(sqlText: string, line: number, column: number): string { return output.join("\n"); } -async function assertSchemaOwnership( - tx: ISql<{}>, - schema: string, -): Promise { +async function assertSchemaOwnership(tx: ISql, schema: string): Promise { const [result] = await tx` select n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner diff --git a/packages/space/migrate/provision.sql b/packages/database/space/migrate/provision.sql similarity index 100% rename from packages/space/migrate/provision.sql rename to packages/database/space/migrate/provision.sql diff --git a/packages/space/migrate/sql.d.ts b/packages/database/space/migrate/sql.d.ts similarity index 100% rename from packages/space/migrate/sql.d.ts rename to packages/database/space/migrate/sql.d.ts diff --git a/packages/space/migrate/test-utils.ts b/packages/database/space/migrate/test-utils.ts similarity index 100% rename from packages/space/migrate/test-utils.ts rename to packages/database/space/migrate/test-utils.ts diff --git a/packages/space/slug.test.ts b/packages/database/space/slug.test.ts similarity index 100% rename from packages/space/slug.test.ts rename to packages/database/space/slug.test.ts diff --git a/packages/space/slug.ts b/packages/database/space/slug.ts similarity index 100% rename from packages/space/slug.ts rename to packages/database/space/slug.ts diff --git a/packages/space/version.ts b/packages/database/space/version.ts similarity index 100% rename from packages/space/version.ts rename to packages/database/space/version.ts diff --git a/packages/core/tsconfig.json b/packages/database/tsconfig.json similarity index 100% rename from packages/core/tsconfig.json rename to packages/database/tsconfig.json diff --git a/packages/space/package.json b/packages/space/package.json deleted file mode 100644 index 7d5322b..0000000 --- a/packages/space/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "@memory.build/space", - "version": "0.2.5", - "private": true, - "type": "module", - "dependencies": { - "@pydantic/logfire-node": "^0.13.1", - "postgres": "^3.4.9" - } -} diff --git a/packages/space/tsconfig.json b/packages/space/tsconfig.json deleted file mode 100644 index 23b1d27..0000000 --- a/packages/space/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "include": ["**/*.ts", "**/*.d.ts"] -} diff --git a/scripts/migrate-db.ts b/scripts/migrate-db.ts index 412d5eb..312e283 100644 --- a/scripts/migrate-db.ts +++ b/scripts/migrate-db.ts @@ -1,11 +1,12 @@ #!/usr/bin/env bun -import { CORE_SCHEMA_VERSION, migrateCore } from "@memory.build/core"; import { bootstrapSpaceDatabase, + CORE_SCHEMA_VERSION, + migrateCore, migrateSpace, SPACE_SCHEMA_VERSION, slugToSchema, -} from "@memory.build/space"; +} from "@memory.build/database"; import postgres from "postgres"; const DEFAULT_DATABASE_URL = "postgresql://postgres@127.0.0.1:5432/postgres"; diff --git a/scripts/package.json b/scripts/package.json index 87c9915..69cefac 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -3,9 +3,8 @@ "private": true, "type": "module", "dependencies": { - "@memory.build/core": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", - "@memory.build/space": "workspace:*", "postgres": "^3.4.9", "yaml": "^2.7.0" } From fdb26e9fed2853b5830219ba75a4def4c370cc0c Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 2 Jun 2026 15:24:18 +0200 Subject: [PATCH 023/156] refactor(database): extract shared test-utils into one internal module The core and space test-utils duplicated ~110 lines of generic, driver-level helpers (connect, resolveTestDatabaseUrl, expectReject, and schema introspection). Move them once into packages/database/migrate/test-utils.ts; core/ and space/ test-utils now `export *` from it and keep only their provisioning (TestCore/TestSpace, randomCoreSchema/randomSlug/testSchema). Test files are unchanged. Verified: typecheck + lint clean; 761 unit tests pass; ghost db integration (core 18, space 19) pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 28 +-- packages/database/core/migrate/test-utils.ts | 152 +------------- packages/database/migrate/test-utils.ts | 198 ++++++++++++++++++ packages/database/space/migrate/test-utils.ts | 189 +---------------- 4 files changed, 222 insertions(+), 345 deletions(-) create mode 100644 packages/database/migrate/test-utils.ts diff --git a/TODO.md b/TODO.md index 1f1050c..86c4819 100644 --- a/TODO.md +++ b/TODO.md @@ -15,20 +15,20 @@ later is cheap if distribution returns. - [ ] Keep `space/` free of `core/` imports (and vice versa) so the re-split escape hatch stays open — worth a Biome `noRestrictedImports` rule to enforce it. -## Consolidate duplicated test-utils (now an internal module) - -`packages/database/core/migrate/test-utils.ts` and -`packages/database/space/migrate/test-utils.ts` duplicate ~110 lines of generic, -driver-level helpers: `resolveTestDatabaseUrl`, `connect`, `expectReject`, and schema -introspection (`schemaExists`, `tableExists`, `listTables`, `listFunctions`, -`listTriggers`, `appliedMigrations`, `getSchemaVersion`). - -- [ ] Move the generic helpers into one shared internal module (e.g. - `packages/database/migrate/test-utils.ts`) imported by both core/ and space/ - tests. Keep package-specific provisioning where it is: `TestCore`/`TestSpace`, - `randomCoreSchema`/`randomSlug`, `columnType`, the index helpers. -- [ ] `engine`/`accounts` carry their own `tableExists`/`schemaExists` copies too; - they're separate packages, so sharing with them still needs a dev-only package +## Consolidate duplicated test-utils + +- [x] Done — the generic, driver-level helpers (`resolveTestDatabaseUrl`, `connect`, + `expectReject`, and schema introspection: `schemaExists`, `tableExists`, + `listTables`, `listFunctions`, `listTriggers`, `extensionInstalled`, + `columnType`, `listIndexes`, `getIndexDef`, `getIndexReloptions`, + `appliedMigrations`, `getSchemaVersion`) now live once in + `packages/database/migrate/test-utils.ts`. `core/migrate/test-utils.ts` and + `space/migrate/test-utils.ts` `export *` from it and add only their + provisioning (`TestCore`/`withTestCore`/`randomCoreSchema`; + `TestSpace`/`withTestSpace`/`randomSlug`/`testSchema`), so the test files are + unchanged. Verified: typecheck/lint clean, unit + ghost db suites pass. +- [ ] `engine`/`accounts` still carry their own `tableExists`/`schemaExists` copies. + They're separate packages, so sharing with them needs a dev-only package (or fold them in during the postgres.js rollout). ## Consolidate the migration runner logic (now an internal module) diff --git a/packages/database/core/migrate/test-utils.ts b/packages/database/core/migrate/test-utils.ts index ca76a36..e645f5f 100644 --- a/packages/database/core/migrate/test-utils.ts +++ b/packages/database/core/migrate/test-utils.ts @@ -1,35 +1,9 @@ import type { Sql as SQL } from "postgres"; -import postgres from "postgres"; import { type MigrateCoreOptions, migrateCore } from "./migrate"; -// --------------------------------------------------------------------------- -// Connection -// --------------------------------------------------------------------------- -// -// Tests run against a real Postgres that has the required extensions -// (citext, ltree, vector, pg_textsearch) and is PG 18+. Two ways to provide -// one: -// -// - local docker (fast iteration): ./bun run pg (then leave TEST_DATABASE_URL unset) -// - ghost (real TigerData stack): TEST_DATABASE_URL="$(ghost connect testing_me)" -// -// Because the core migrations are templated (production uses the "core" -// schema; tests pass a unique schema name), every test can provision its own -// throwaway core and run concurrently — exactly like space tests, and without -// ever touching a real `core` schema. - -const DEFAULT_TEST_DATABASE_URL = - "postgresql://postgres@127.0.0.1:5432/postgres"; - -export function resolveTestDatabaseUrl(): string { - return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; -} - -export function connect(max = 10): SQL { - // onnotice silences the routine "… already exists, skipping" NOTICEs that the - // idempotent migrations emit (postgres-js prints them to the console by default). - return postgres(resolveTestDatabaseUrl(), { max, onnotice: () => {} }); -} +// Connection, failure assertions, and schema introspection are shared with the +// space suite. +export * from "../../migrate/test-utils"; const SCHEMA_SUFFIX_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; @@ -44,6 +18,10 @@ export function randomCoreSchema(): string { // --------------------------------------------------------------------------- // TestCore — a provisioned, isolated core schema // --------------------------------------------------------------------------- +// +// The core migrations are templated (production uses the "core" schema; tests +// pass a unique throwaway name), so every test gets its own isolated core and +// they run concurrently without ever touching a real `core` schema. export class TestCore { readonly schema: string; @@ -84,119 +62,3 @@ export async function withTestCore( await core.drop(); } } - -/** - * Assert that a query rejects. bun:test's `expect(...).rejects` does not drive - * postgres-js's lazy `PendingQuery` (it only runs when truly awaited), so it - * hangs; awaiting inside try/catch executes the query and observes the error. - */ -export async function expectReject(fn: () => Promise): Promise { - let rejected = false; - try { - await fn(); - } catch { - rejected = true; - } - if (!rejected) { - throw new Error("expected the operation to reject, but it resolved"); - } -} - -// --------------------------------------------------------------------------- -// Schema introspection -// --------------------------------------------------------------------------- - -export async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.schemata where schema_name = ${name} - ) as exists - `; - return Boolean(row?.exists); -} - -export async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = ${table} - ) as exists - `; - return Boolean(row?.exists); -} - -export async function listTables(sql: SQL, schema: string): Promise { - const rows = await sql` - select table_name - from information_schema.tables - where table_schema = ${schema} and table_type = 'BASE TABLE' - order by table_name - `; - return rows.map((r) => r.table_name as string); -} - -/** Distinct function names in a schema (overloads collapse to one entry). */ -export async function listFunctions( - sql: SQL, - schema: string, -): Promise { - const rows = await sql` - select distinct p.proname - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = ${schema} - order by p.proname - `; - return rows.map((r) => r.proname as string); -} - -export async function listTriggers( - sql: SQL, - schema: string, - table: string, -): Promise { - const rows = await sql` - select t.tgname - from pg_trigger t - join pg_class c on c.oid = t.tgrelid - join pg_namespace n on n.oid = c.relnamespace - where n.nspname = ${schema} and c.relname = ${table} and not t.tgisinternal - order by t.tgname - `; - return rows.map((r) => r.tgname as string); -} - -/** Whether an extension is installed (in any schema). */ -export async function extensionInstalled( - sql: SQL, - name: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 from pg_extension where extname = ${name} - ) as exists - `; - return Boolean(row?.exists); -} - -export async function appliedMigrations( - sql: SQL, - schema: string, -): Promise { - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - return rows.map((r) => r.name as string); -} - -export async function getSchemaVersion( - sql: SQL, - schema: string, -): Promise { - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row?.version; -} diff --git a/packages/database/migrate/test-utils.ts b/packages/database/migrate/test-utils.ts new file mode 100644 index 0000000..68dcbb5 --- /dev/null +++ b/packages/database/migrate/test-utils.ts @@ -0,0 +1,198 @@ +import type { Sql as SQL } from "postgres"; +import postgres from "postgres"; + +// --------------------------------------------------------------------------- +// Shared test helpers for the database package's migration suites +// --------------------------------------------------------------------------- +// +// Generic, schema-agnostic helpers used by both the core and space test-utils +// (which re-export this module and add their own provisioning). Tests run +// against a real PostgreSQL 18 with citext/ltree/vector/pg_textsearch; point +// `TEST_DATABASE_URL` at a ghost database (see CLAUDE.md → Database integration +// tests). Falls back to a local Postgres when unset. + +const DEFAULT_TEST_DATABASE_URL = + "postgresql://postgres@127.0.0.1:5432/postgres"; + +export function resolveTestDatabaseUrl(): string { + return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; +} + +export function connect(max = 10): SQL { + // onnotice silences the routine "… already exists, skipping" NOTICEs that the + // idempotent migrations emit (postgres-js prints them to the console by default). + return postgres(resolveTestDatabaseUrl(), { max, onnotice: () => {} }); +} + +/** + * Assert that a query rejects. bun:test's `expect(...).rejects` does not drive + * postgres-js's lazy `PendingQuery` (it only runs when truly awaited), so it + * hangs; awaiting inside try/catch executes the query and observes the error. + */ +export async function expectReject(fn: () => Promise): Promise { + let rejected = false; + try { + await fn(); + } catch { + rejected = true; + } + if (!rejected) { + throw new Error("expected the operation to reject, but it resolved"); + } +} + +// --------------------------------------------------------------------------- +// Schema introspection +// --------------------------------------------------------------------------- + +export async function schemaExists(sql: SQL, name: string): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${name} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function tableExists( + sql: SQL, + schema: string, + table: string, +): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.tables + where table_schema = ${schema} and table_name = ${table} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function listTables(sql: SQL, schema: string): Promise { + const rows = await sql` + select table_name + from information_schema.tables + where table_schema = ${schema} and table_type = 'BASE TABLE' + order by table_name + `; + return rows.map((r) => r.table_name as string); +} + +/** Distinct function names in a schema (overloads collapse to one entry). */ +export async function listFunctions( + sql: SQL, + schema: string, +): Promise { + const rows = await sql` + select distinct p.proname + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = ${schema} + order by p.proname + `; + return rows.map((r) => r.proname as string); +} + +export async function listTriggers( + sql: SQL, + schema: string, + table: string, +): Promise { + const rows = await sql` + select t.tgname + from pg_trigger t + join pg_class c on c.oid = t.tgrelid + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = ${schema} and c.relname = ${table} and not t.tgisinternal + order by t.tgname + `; + return rows.map((r) => r.tgname as string); +} + +/** Whether an extension is installed (in any schema). */ +export async function extensionInstalled( + sql: SQL, + name: string, +): Promise { + const [row] = await sql` + select exists ( + select 1 from pg_extension where extname = ${name} + ) as exists + `; + return Boolean(row?.exists); +} + +/** Fully-resolved type of a column, e.g. "halfvec(768)" or "uuid". */ +export async function columnType( + sql: SQL, + schema: string, + table: string, + column: string, +): Promise { + const [row] = await sql` + select format_type(a.atttypid, a.atttypmod) as type + from pg_attribute a + where a.attrelid = ${`${schema}.${table}`}::regclass + and a.attname = ${column} + and not a.attisdropped + `; + return row?.type ?? null; +} + +export async function listIndexes( + sql: SQL, + schema: string, + table: string, +): Promise { + const rows = await sql` + select indexname from pg_indexes + where schemaname = ${schema} and tablename = ${table} + order by indexname + `; + return rows.map((r) => r.indexname as string); +} + +export async function getIndexDef( + sql: SQL, + schema: string, + index: string, +): Promise { + const [row] = await sql` + select indexdef from pg_indexes + where schemaname = ${schema} and indexname = ${index} + `; + return row?.indexdef ?? null; +} + +/** Storage parameters of an index, e.g. ["m=8", "ef_construction=32"]. */ +export async function getIndexReloptions( + sql: SQL, + schema: string, + index: string, +): Promise { + const [row] = await sql` + select c.reloptions + from pg_class c + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = ${schema} and c.relname = ${index} + `; + return row?.reloptions ?? []; +} + +export async function appliedMigrations( + sql: SQL, + schema: string, +): Promise { + const rows = await sql.unsafe( + `select name from ${schema}.migration order by name`, + ); + return rows.map((r) => r.name as string); +} + +export async function getSchemaVersion( + sql: SQL, + schema: string, +): Promise { + const [row] = await sql.unsafe(`select version from ${schema}.version`); + return row?.version; +} diff --git a/packages/database/space/migrate/test-utils.ts b/packages/database/space/migrate/test-utils.ts index 77ebed9..88dbf41 100644 --- a/packages/database/space/migrate/test-utils.ts +++ b/packages/database/space/migrate/test-utils.ts @@ -1,31 +1,9 @@ import type { Sql as SQL } from "postgres"; -import postgres from "postgres"; import { type MigrateSpaceOptions, migrateSpace } from "./migrate"; -// --------------------------------------------------------------------------- -// Connection -// --------------------------------------------------------------------------- -// -// See packages/core/migrate/test-utils.ts for the connection model. In short: -// provide a PG 18+ database with citext/ltree/vector/pg_textsearch via -// TEST_DATABASE_URL (ghost) or fall back to local docker Postgres. -// -// Spaces isolate cleanly: each space is its own `me_` schema, so many -// can coexist in one database and tests can run concurrently with unique -// slugs — no `create database` needed (ghost forbids it anyway). - -const DEFAULT_TEST_DATABASE_URL = - "postgresql://postgres@127.0.0.1:5432/postgres"; - -export function resolveTestDatabaseUrl(): string { - return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; -} - -export function connect(max = 10): SQL { - // onnotice silences the routine "… already exists, skipping" NOTICEs that the - // idempotent migrations emit (postgres-js prints them to the console by default). - return postgres(resolveTestDatabaseUrl(), { max, onnotice: () => {} }); -} +// Connection, failure assertions, and schema introspection are shared with the +// core suite. +export * from "../../migrate/test-utils"; /** * Test spaces provision under a `metest_` schema instead of the production @@ -105,164 +83,3 @@ export async function withTestSpace( await space.drop(); } } - -/** - * Assert that a query rejects. bun:test's `expect(...).rejects` does not drive - * postgres-js's lazy `PendingQuery` (it only runs when truly awaited), so it - * hangs; awaiting inside try/catch executes the query and observes the error. - */ -export async function expectReject(fn: () => Promise): Promise { - let rejected = false; - try { - await fn(); - } catch { - rejected = true; - } - if (!rejected) { - throw new Error("expected the operation to reject, but it resolved"); - } -} - -// --------------------------------------------------------------------------- -// Schema introspection (mirrors packages/core/migrate/test-utils.ts; kept -// local so the package stays self-contained) -// --------------------------------------------------------------------------- - -export async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.schemata where schema_name = ${name} - ) as exists - `; - return Boolean(row?.exists); -} - -export async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = ${table} - ) as exists - `; - return Boolean(row?.exists); -} - -export async function listTables(sql: SQL, schema: string): Promise { - const rows = await sql` - select table_name - from information_schema.tables - where table_schema = ${schema} and table_type = 'BASE TABLE' - order by table_name - `; - return rows.map((r) => r.table_name as string); -} - -/** Distinct function names in a schema (overloads collapse to one entry). */ -export async function listFunctions( - sql: SQL, - schema: string, -): Promise { - const rows = await sql` - select distinct p.proname - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = ${schema} - order by p.proname - `; - return rows.map((r) => r.proname as string); -} - -export async function listTriggers( - sql: SQL, - schema: string, - table: string, -): Promise { - const rows = await sql` - select t.tgname - from pg_trigger t - join pg_class c on c.oid = t.tgrelid - join pg_namespace n on n.oid = c.relnamespace - where n.nspname = ${schema} and c.relname = ${table} and not t.tgisinternal - order by t.tgname - `; - return rows.map((r) => r.tgname as string); -} - -/** Fully-resolved type of a column, e.g. "halfvec(768)" or "uuid". */ -export async function columnType( - sql: SQL, - schema: string, - table: string, - column: string, -): Promise { - const [row] = await sql` - select format_type(a.atttypid, a.atttypmod) as type - from pg_attribute a - where a.attrelid = ${`${schema}.${table}`}::regclass - and a.attname = ${column} - and not a.attisdropped - `; - return row?.type ?? null; -} - -export async function listIndexes( - sql: SQL, - schema: string, - table: string, -): Promise { - const rows = await sql` - select indexname from pg_indexes - where schemaname = ${schema} and tablename = ${table} - order by indexname - `; - return rows.map((r) => r.indexname as string); -} - -export async function getIndexDef( - sql: SQL, - schema: string, - index: string, -): Promise { - const [row] = await sql` - select indexdef from pg_indexes - where schemaname = ${schema} and indexname = ${index} - `; - return row?.indexdef ?? null; -} - -/** Storage parameters of an index, e.g. ["m=8", "ef_construction=32"]. */ -export async function getIndexReloptions( - sql: SQL, - schema: string, - index: string, -): Promise { - const [row] = await sql` - select c.reloptions - from pg_class c - join pg_namespace n on n.oid = c.relnamespace - where n.nspname = ${schema} and c.relname = ${index} - `; - return row?.reloptions ?? []; -} - -export async function appliedMigrations( - sql: SQL, - schema: string, -): Promise { - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - return rows.map((r) => r.name as string); -} - -export async function getSchemaVersion( - sql: SQL, - schema: string, -): Promise { - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row?.version; -} From c4d99c7da0dff1e8f86e931e171bf898cdeac972 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 2 Jun 2026 17:45:36 +0200 Subject: [PATCH 024/156] refactor(database): extract shared migration runner into migrate/kit.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit migrateCore, migrateSpace, and bootstrapSpaceDatabase duplicated nearly all of the migration machinery. Move it once into packages/database/migrate/kit.ts: advisory locking, session timeouts, extension/Postgres-version preconditions, schema checks (exists/ownership/valid-name), {{…}} templating, SQL-file execution with error-location logging, and the incremental-once/idempotent-always runSchemaMigrations runner. The kit is parameterized by a `label` (drives span/attribute/log names, so telemetry keys are unchanged) and `dir`; the three entry points are now thin orchestrators holding only their schema-specific bits. bootstrap's advisory lock moves from a hardcoded single-key id to the shared two-key derived lock (internal; bootstrap is its only user). Verified: typecheck + lint clean; ghost db integration (core 18, space 19) pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 34 +- packages/database/core/migrate/migrate.ts | 435 +++---------------- packages/database/migrate/kit.ts | 418 ++++++++++++++++++ packages/database/space/migrate/bootstrap.ts | 127 +----- packages/database/space/migrate/migrate.ts | 371 ++-------------- 5 files changed, 552 insertions(+), 833 deletions(-) create mode 100644 packages/database/migrate/kit.ts diff --git a/TODO.md b/TODO.md index 86c4819..a233f89 100644 --- a/TODO.md +++ b/TODO.md @@ -31,21 +31,19 @@ later is cheap if distribution returns. They're separate packages, so sharing with them needs a dev-only package (or fold them in during the postgres.js rollout). -## Consolidate the migration runner logic (now an internal module) - -`packages/database/core/migrate/migrate.ts`, `.../space/migrate/migrate.ts`, and -`.../space/migrate/bootstrap.ts` duplicate most of the migration machinery: - -- advisory locking (`advisoryLockKey`, `acquireAdvisoryLock`, retry/backoff) -- SQL-file execution with error-location logging (`executeSqlFile`, - `logPostgresSqlLocation`, `sqlLocation`, `sqlContext`) -- extension / Postgres-version preconditions (`ensureExtension`, - `ensurePostgresVersion`, `REQUIRED_EXTENSIONS`) -- `{{template}}` rendering -- the incremental-once / idempotent-always runner, with version + migration tracking -- telemetry span wrapping - -- [ ] Extract into a shared internal module (e.g. `packages/database/migrate/_kit.ts`) - parameterized by schema name, the ordered incremental/idempotent SQL lists, and - template vars. `migrateCore` / `migrateSpace` / `bootstrapSpaceDatabase` become - thin callers, leaving each `migrate.ts` with only its schema-specific bits. +## Consolidate the migration runner logic + +- [x] Done — the shared machinery lives in `packages/database/migrate/kit.ts`: + advisory locking (`advisoryLockKey`, `acquireAdvisoryLock`), session timeouts + (`applySessionTimeouts`), extension / Postgres-version preconditions + (`ensurePostgresVersion`, `ensureExtension`, `ensureRequiredExtensions`, + `REQUIRED_EXTENSIONS`), schema checks (`doesSchemaExist`, + `assertSchemaOwnership`, `isValidSchemaName`), `{{…}}` `template` rendering, + SQL-file execution with error-location logging (`executeSqlFile`), and the + incremental-once / idempotent-always `runSchemaMigrations` runner — all + parameterized by a `label` (drives span/attribute/log names) + `dir`. + `migrateCore` / `migrateSpace` / `bootstrapSpaceDatabase` are now thin + orchestrators holding only their schema-specific bits (options, SQL lists, + slug/shard handling, template vars). Verified: typecheck/lint clean, + unit + ghost db suites pass. (bootstrap's lock moved from a hardcoded + single-key id to the shared two-key derived lock.) diff --git a/packages/database/core/migrate/migrate.ts b/packages/database/core/migrate/migrate.ts index 198409b..b09fe0e 100644 --- a/packages/database/core/migrate/migrate.ts +++ b/packages/database/core/migrate/migrate.ts @@ -1,8 +1,31 @@ -import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; import { semver } from "bun"; -import type { ISql, Sql as SQL } from "postgres"; +import type { Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + doesSchemaExist, + ensurePostgresVersion, + ensureRequiredExtensions, + executeSqlFile, + isValidSchemaName, + type Migration, + REQUIRED_EXTENSIONS, + runSchemaMigrations, + template, +} from "../../migrate/kit"; import { CORE_SCHEMA_VERSION } from "../version"; +import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; +import idempotent001 from "./idempotent/001_principal_space.sql" with { + type: "text", +}; +import idempotent002 from "./idempotent/002_group_member.sql" with { + type: "text", +}; +import idempotent003 from "./idempotent/003_tree_access.sql" with { + type: "text", +}; import incremental001 from "./incremental/001_space.sql" with { type: "text" }; import incremental002 from "./incremental/002_principal.sql" with { type: "text", @@ -21,13 +44,9 @@ import incremental006 from "./incremental/006_api_key.sql" with { }; import provisionSql from "./provision.sql" with { type: "text" }; -interface Incremental { - name: string; - file: string; - sql: string; -} +const DIR = "packages/database/core/migrate"; -const incrementals: Incremental[] = [ +const incrementals: Migration[] = [ { name: "001_space", file: "incremental/001_space.sql", sql: incremental001 }, { name: "002_principal", @@ -56,24 +75,7 @@ const incrementals: Incremental[] = [ }, ]; -import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; -import idempotent001 from "./idempotent/001_principal_space.sql" with { - type: "text", -}; -import idempotent002 from "./idempotent/002_group_member.sql" with { - type: "text", -}; -import idempotent003 from "./idempotent/003_tree_access.sql" with { - type: "text", -}; - -interface Idempotent { - name: string; - file: string; - sql: string; -} - -const idempotents: Idempotent[] = [ +const idempotents: Migration[] = [ { name: "000_update", file: "idempotent/000_update.sql", sql: idempotent000 }, { name: "001_principal_space", @@ -100,13 +102,6 @@ const idempotents: Idempotent[] = [ */ export const CORE_SCHEMA = "core"; -const REQUIRED_EXTENSIONS = [ - { name: "citext", minVersion: "1.6" }, - { name: "ltree", minVersion: "1.3" }, - { name: "vector", minVersion: "0.8.2" }, - { name: "pg_textsearch", minVersion: "1.1.0" }, -] as const; - export interface MigrateCoreOptions { schema?: string; logSqlFiles?: boolean; @@ -150,10 +145,7 @@ export async function migrateCore( ); await sql.begin(async (tx) => { - await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; - await tx`select set_config('lock_timeout', ${opts.lockTimeout}, true)`; - await tx`select set_config('transaction_timeout', ${opts.transactionTimeout}, true)`; - await tx`select set_config('idle_in_transaction_session_timeout', ${opts.idleInTransactionSessionTimeout}, true)`; + await applySessionTimeouts(tx, opts); const acquired = await span("core.migrate.acquire_lock", { attributes, callback: () => acquireAdvisoryLock(tx, key1, key2), @@ -163,31 +155,44 @@ export async function migrateCore( } await ensurePostgresVersion(tx); - for (const extension of REQUIRED_EXTENSIONS) { - await span("core.migrate.ensure_extension", { - attributes: { - "db.extension": extension.name, - "db.extension_min_version": extension.minVersion, - }, - callback: () => - ensureExtension(tx, extension.name, extension.minVersion), - }); - } + await ensureRequiredExtensions(tx, "core.migrate"); - if (!(await doesCoreExist(tx, opts.schema))) { + if (!(await doesSchemaExist(tx, opts.schema))) { await span("core.migrate.provision", { attributes: { ...attributes, "core.migration_file": "provision.sql", "core.migration_type": "provision", }, - callback: () => provisionCore(tx, opts), + callback: () => + executeSqlFile( + tx, + template(provisionSql, { schema: opts.schema }), + { + logSqlFiles: opts.logSqlFiles, + label: "core", + schema: opts.schema, + type: "provision", + dir: DIR, + file: "provision.sql", + }, + ), }); info("Core schema provisioned", attributes); } await span("core.migrate.run", { attributes, - callback: () => runMigrations(tx, opts), + callback: () => + runSchemaMigrations(tx, { + schema: opts.schema, + schemaVersion: opts.schemaVersion, + incrementals, + idempotents, + templateVars: { schema: opts.schema }, + label: "core", + dir: DIR, + logSqlFiles: opts.logSqlFiles, + }), }); }); info("Core migrations completed", attributes); @@ -230,333 +235,3 @@ function normalizeMigrateCoreOptions( options.idleInTransactionSessionTimeout ?? "5s", }; } - -function isValidSchemaName(schema: string): boolean { - return /^[a-z_][a-z0-9_]*$/.test(schema) && schema.length <= 63; -} - -function advisoryLockKey(schema: string): [number, number] { - const digest = createHash("sha256").update(schema).digest(); - return [digest.readInt32BE(0), digest.readInt32BE(4)]; -} - -const MAX_LOCK_RETRIES = 5; -const BASE_DELAY_MS = 100; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function acquireAdvisoryLock( - tx: ISql, - key1: number, - key2: number, -): Promise { - let acquired = false; - for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { - const [result] = await tx` - select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired - `; - if (result?.acquired) { - acquired = true; - break; - } - if (attempt < MAX_LOCK_RETRIES - 1) { - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } - return acquired; -} - -async function doesCoreExist(tx: ISql, schema: string): Promise { - const [row] = await tx` - select exists - ( - select 1 - from pg_namespace n - where n.nspname = ${schema} - ) as "coreExists" - `; - return Boolean(row?.coreExists); -} - -async function provisionCore( - tx: ISql, - options: NormalizedMigrateCoreOptions, -): Promise { - await executeSqlFile( - tx, - options, - "provision", - "provision.sql", - template(provisionSql, { schema: options.schema }), - ); -} - -async function ensurePostgresVersion(tx: ISql): Promise { - const [row] = await tx` - select current_setting('server_version_num')::int as server_version_num - `; - const server_version_num = Number(row?.server_version_num); - if (server_version_num < 180000) { - throw new Error( - `PostgreSQL version 18 or higher is required (found ${server_version_num})`, - ); - } -} - -async function ensureExtension( - tx: ISql, - name: string, - minVersion: string, -): Promise { - const [installed] = await tx` - select x.extversion, n.nspname - from pg_extension x - inner join pg_namespace n on (x.extnamespace = n.oid) - where x.extname = ${name} - `; - - if (installed) { - if ( - installed.nspname === "public" && - semver.order(installed.extversion, minVersion) >= 0 - ) { - return; - } - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required in the "public" schema (found ${installed.extversion} installed in "${installed.nspname}")`, - ); - } - - const [available] = await tx` - select default_version - from pg_available_extensions - where name = ${name} - `; - - if (!available || semver.order(available.default_version, minVersion) < 0) { - const found = available - ? `found ${available.default_version} available` - : "not available"; - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required (${found})`, - ); - } - - await tx`create extension if not exists ${tx(name)} with schema public`; -} - -async function runMigrations( - tx: ISql, - options: NormalizedMigrateCoreOptions, -): Promise { - const schema = options.schema; - await assertSchemaOwnership(tx, schema); - - const [versionRow] = await tx` - select version from ${tx(schema)}.version - `; - const dbVersion: string = versionRow?.version; - const cmp = semver.order(options.schemaVersion, dbVersion); - if (cmp < 0) { - throw new Error( - `Application version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the application.", - ); - } - /* run migrations regardless - if (cmp === 0) { - info("Core migration skipped, version current", { - "db.schema": schema, - "core.version": dbVersion, - "core.schema_version": options.schemaVersion, - }); - return; - } - */ - - const sorted1 = [...incrementals].sort((a, b) => - a.name.localeCompare(b.name), - ); - - for (const migration of sorted1) { - const [existingRow] = await tx` - select exists - ( - select 1 - from ${tx(schema)}.migration - where name = ${migration.name} - ) as existing - `; - - if (existingRow?.existing) { - continue; - } - - await span("core.migrate.incremental", { - attributes: { - "db.schema": schema, - "core.migration": migration.name, - "core.migration_file": migration.file, - "core.migration_type": "incremental", - "core.schema_version": options.schemaVersion, - }, - callback: async () => { - await executeSqlFile( - tx, - options, - "incremental", - migration.file, - template(migration.sql, { schema }), - ); - await tx` - insert into ${tx(schema)}.migration (name, applied_at_version) - values (${migration.name}, ${options.schemaVersion})`; - }, - }); - info("Core migration applied", { - "db.schema": schema, - "core.migration": migration.name, - "core.migration_file": migration.file, - "core.migration_type": "incremental", - "core.schema_version": options.schemaVersion, - }); - } - - const sorted2 = [...idempotents].sort((a, b) => a.name.localeCompare(b.name)); - - for (const migration of sorted2) { - await span("core.migrate.idempotent", { - attributes: { - "db.schema": schema, - "core.migration": migration.name, - "core.migration_file": migration.file, - "core.migration_type": "idempotent", - "core.schema_version": options.schemaVersion, - }, - callback: () => - executeSqlFile( - tx, - options, - "idempotent", - migration.file, - template(migration.sql, { schema }), - ), - }); - } - - await tx`update ${tx(schema)}.version set version = ${options.schemaVersion}, at = now()`; -} - -async function executeSqlFile( - tx: ISql, - options: NormalizedMigrateCoreOptions, - type: string, - file: string, - sqlText: string, -): Promise { - logSqlFile(options, type, file); - try { - await tx.unsafe(sqlText); - } catch (error) { - logSqlExecutionError(options, type, file, sqlText, error); - throw error; - } -} - -function logSqlFile( - options: NormalizedMigrateCoreOptions, - type: string, - file: string, -): void { - if (!options.logSqlFiles) return; - console.error(`[migrate:db] core ${type} packages/core/migrate/${file}`); -} - -function logSqlExecutionError( - options: NormalizedMigrateCoreOptions, - type: string, - file: string, - sqlText: string, - error: unknown, -): void { - if (!options.logSqlFiles) return; - console.error( - `[migrate:db] failed core ${type} packages/core/migrate/${file}`, - ); - logPostgresSqlLocation(sqlText, error); -} - -function logPostgresSqlLocation(sqlText: string, error: unknown): void { - // postgres-js sets `position` (1-based) on server errors; non-PG errors won't. - const position = Number((error as { position?: unknown })?.position); - if (!Number.isSafeInteger(position) || position < 1) return; - - const location = sqlLocation(sqlText, position); - if (!location) return; - console.error( - `[migrate:db] sql position ${position} -> line ${location.line}, column ${location.column}`, - ); - console.error(sqlContext(sqlText, location.line, location.column)); -} - -function sqlLocation( - sqlText: string, - position: number, -): { line: number; column: number } | undefined { - if (position > sqlText.length + 1) return undefined; - let line = 1; - let column = 1; - for (let i = 0; i < position - 1; i++) { - if (sqlText.charCodeAt(i) === 10) { - line++; - column = 1; - } else { - column++; - } - } - return { line, column }; -} - -function sqlContext(sqlText: string, line: number, column: number): string { - const lines = sqlText.split("\n"); - const start = Math.max(1, line - 2); - const end = Math.min(lines.length, line + 2); - const width = String(end).length; - const output = ["[migrate:db] sql context:"]; - - for (let n = start; n <= end; n++) { - const marker = n === line ? ">" : " "; - output.push(`${marker} ${String(n).padStart(width)} | ${lines[n - 1]}`); - if (n === line) { - output.push(` ${" ".repeat(width)} | ${" ".repeat(column - 1)}^`); - } - } - - return output.join("\n"); -} - -async function assertSchemaOwnership(tx: ISql, schema: string): Promise { - const [result] = await tx` - select - n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner - from pg_catalog.pg_namespace n - where n.nspname = ${schema} - `; - - if (!result?.is_owner) { - throw new Error( - `Only the owner of the ${schema} schema can run database migrations`, - ); - } -} - -function template(sql: string, vars: Record): string { - return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { - if (!(key in vars)) { - throw new Error(`Missing template variable: ${key}`); - } - return String(vars[key]); - }); -} diff --git a/packages/database/migrate/kit.ts b/packages/database/migrate/kit.ts new file mode 100644 index 0000000..90f233f --- /dev/null +++ b/packages/database/migrate/kit.ts @@ -0,0 +1,418 @@ +import { createHash } from "node:crypto"; +import { info, span } from "@pydantic/logfire-node"; +import { semver } from "bun"; +import type { ISql } from "postgres"; + +// --------------------------------------------------------------------------- +// Shared migration machinery for the core (control plane) and space (data +// plane) migrators. migrateCore / migrateSpace / bootstrapSpaceDatabase are +// thin orchestrators over these helpers. `label` (e.g. "core" / "space") drives +// span names, telemetry attribute keys, and log messages so each migrator keeps +// its existing observability; `dir` is the on-disk path used in SQL-file logs. +// --------------------------------------------------------------------------- + +export interface Migration { + name: string; + file: string; + sql: string; +} + +export const REQUIRED_EXTENSIONS = [ + { name: "citext", minVersion: "1.6" }, + { name: "ltree", minVersion: "1.3" }, + { name: "vector", minVersion: "0.8.2" }, + { name: "pg_textsearch", minVersion: "1.1.0" }, +] as const; + +/** A valid lowercase SQL identifier usable as a schema name (<= 63 chars). */ +export function isValidSchemaName(schema: string): boolean { + return /^[a-z_][a-z0-9_]*$/.test(schema) && schema.length <= 63; +} + +// --------------------------------------------------------------------------- +// Advisory locking +// --------------------------------------------------------------------------- + +const MAX_LOCK_RETRIES = 5; +const BASE_DELAY_MS = 100; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Derive a stable (int4, int4) advisory-lock key pair from a name. */ +export function advisoryLockKey(name: string): [number, number] { + const digest = createHash("sha256").update(name).digest(); + return [digest.readInt32BE(0), digest.readInt32BE(4)]; +} + +/** Try to take a transaction-scoped advisory lock, with bounded backoff. */ +export async function acquireAdvisoryLock( + tx: ISql, + key1: number, + key2: number, +): Promise { + for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { + const [result] = await tx` + select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired + `; + if (result?.acquired) return true; + if (attempt < MAX_LOCK_RETRIES - 1) { + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } + return false; +} + +// --------------------------------------------------------------------------- +// Session / precondition helpers +// --------------------------------------------------------------------------- + +export interface SessionTimeouts { + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function applySessionTimeouts( + tx: ISql, + t: SessionTimeouts, +): Promise { + await tx`select set_config('statement_timeout', ${t.statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${t.lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${t.transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${t.idleInTransactionSessionTimeout}, true)`; +} + +export async function ensurePostgresVersion(tx: ISql): Promise { + const [row] = await tx` + select current_setting('server_version_num')::int as server_version_num + `; + const serverVersionNum = Number(row?.server_version_num); + if (serverVersionNum < 180000) { + throw new Error( + `PostgreSQL version 18 or higher is required (found ${serverVersionNum})`, + ); + } +} + +export async function ensureExtension( + tx: ISql, + name: string, + minVersion: string, +): Promise { + const [installed] = await tx` + select x.extversion, n.nspname + from pg_extension x + inner join pg_namespace n on (x.extnamespace = n.oid) + where x.extname = ${name} + `; + + if (installed) { + if ( + installed.nspname === "public" && + semver.order(installed.extversion, minVersion) >= 0 + ) { + return; + } + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required in the "public" schema (found ${installed.extversion} installed in "${installed.nspname}")`, + ); + } + + const [available] = await tx` + select default_version + from pg_available_extensions + where name = ${name} + `; + + if (!available || semver.order(available.default_version, minVersion) < 0) { + const found = available + ? `found ${available.default_version} available` + : "not available"; + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required (${found})`, + ); + } + + await tx`create extension if not exists ${tx(name)} with schema public`; +} + +/** Ensure every REQUIRED_EXTENSIONS entry, each wrapped in a span. */ +export async function ensureRequiredExtensions( + tx: ISql, + spanPrefix: string, +): Promise { + for (const extension of REQUIRED_EXTENSIONS) { + await span(`${spanPrefix}.ensure_extension`, { + attributes: { + "db.extension": extension.name, + "db.extension_min_version": extension.minVersion, + }, + callback: () => ensureExtension(tx, extension.name, extension.minVersion), + }); + } +} + +export async function doesSchemaExist( + tx: ISql, + schema: string, +): Promise { + const [row] = await tx` + select exists ( + select 1 from pg_namespace n where n.nspname = ${schema} + ) as present + `; + return Boolean(row?.present); +} + +export async function assertSchemaOwnership( + tx: ISql, + schema: string, +): Promise { + const [result] = await tx` + select + n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner + from pg_catalog.pg_namespace n + where n.nspname = ${schema} + `; + + if (!result?.is_owner) { + throw new Error( + `Only the owner of the ${schema} schema can run database migrations`, + ); + } +} + +// --------------------------------------------------------------------------- +// Templating +// --------------------------------------------------------------------------- + +/** Substitute `{{name}}` placeholders; throws on an unknown placeholder. */ +export function template(sql: string, vars: Record): string { + return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { + if (!(key in vars)) { + throw new Error(`Missing template variable: ${key}`); + } + return String(vars[key]); + }); +} + +// --------------------------------------------------------------------------- +// SQL-file execution (with error-location logging) +// --------------------------------------------------------------------------- + +export interface SqlFileContext { + logSqlFiles: boolean; + label: string; + schema: string; + type: string; // "provision" | "incremental" | "idempotent" + dir: string; // e.g. "packages/database/core/migrate" + file: string; // e.g. "incremental/001_space.sql" +} + +export async function executeSqlFile( + tx: ISql, + sqlText: string, + ctx: SqlFileContext, +): Promise { + if (ctx.logSqlFiles) { + console.error( + `[migrate:db] ${ctx.label} ${ctx.schema} ${ctx.type} ${ctx.dir}/${ctx.file}`, + ); + } + try { + await tx.unsafe(sqlText); + } catch (error) { + if (ctx.logSqlFiles) { + console.error( + `[migrate:db] failed ${ctx.label} ${ctx.schema} ${ctx.type} ${ctx.dir}/${ctx.file}`, + ); + logPostgresSqlLocation(sqlText, error); + } + throw error; + } +} + +function logPostgresSqlLocation(sqlText: string, error: unknown): void { + // postgres-js sets `position` (1-based) on server errors; non-PG errors won't. + const position = Number((error as { position?: unknown })?.position); + if (!Number.isSafeInteger(position) || position < 1) return; + + const location = sqlLocation(sqlText, position); + if (!location) return; + console.error( + `[migrate:db] sql position ${position} -> line ${location.line}, column ${location.column}`, + ); + console.error(sqlContext(sqlText, location.line, location.column)); +} + +function sqlLocation( + sqlText: string, + position: number, +): { line: number; column: number } | undefined { + if (position > sqlText.length + 1) return undefined; + let line = 1; + let column = 1; + for (let i = 0; i < position - 1; i++) { + if (sqlText.charCodeAt(i) === 10) { + line++; + column = 1; + } else { + column++; + } + } + return { line, column }; +} + +function sqlContext(sqlText: string, line: number, column: number): string { + const lines = sqlText.split("\n"); + const start = Math.max(1, line - 2); + const end = Math.min(lines.length, line + 2); + const width = String(end).length; + const output = ["[migrate:db] sql context:"]; + + for (let n = start; n <= end; n++) { + const marker = n === line ? ">" : " "; + output.push(`${marker} ${String(n).padStart(width)} | ${lines[n - 1]}`); + if (n === line) { + output.push(` ${" ".repeat(width)} | ${" ".repeat(column - 1)}^`); + } + } + + return output.join("\n"); +} + +// --------------------------------------------------------------------------- +// The incremental-once / idempotent-always runner +// --------------------------------------------------------------------------- + +export interface RunMigrationsConfig { + schema: string; + schemaVersion: string; + incrementals: Migration[]; + idempotents: Migration[]; + /** Template vars applied to every migration's SQL (always includes `schema`). */ + templateVars: Record; + label: string; // "core" | "space" + dir: string; + logSqlFiles: boolean; +} + +function migrationAttributes( + label: string, + schema: string, + schemaVersion: string, + migration: Migration, + type: string, +): Record { + return { + "db.schema": schema, + [`${label}.migration`]: migration.name, + [`${label}.migration_file`]: migration.file, + [`${label}.migration_type`]: type, + [`${label}.schema_version`]: schemaVersion, + }; +} + +/** + * Assumes the schema's version + migration tracking tables exist (created by + * provision). Checks ownership, rejects downgrades, applies pending incremental + * migrations once (tracked), re-applies all idempotent migrations, and stamps + * the version. + */ +export async function runSchemaMigrations( + tx: ISql, + cfg: RunMigrationsConfig, +): Promise { + const { schema, schemaVersion, label, dir, logSqlFiles, templateVars } = cfg; + const Label = label.charAt(0).toUpperCase() + label.slice(1); + + await assertSchemaOwnership(tx, schema); + + const [versionRow] = await tx`select version from ${tx(schema)}.version`; + const dbVersion: string = versionRow?.version; + const cmp = semver.order(schemaVersion, dbVersion); + // abort if the application is older than the database + if (cmp < 0) { + throw new Error( + `Application version (${schemaVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the application.", + ); + } + /* run migrations regardless + if (cmp === 0) { + // version matches; no need to run migrations + info(`${Label} migration skipped, version current`, { + "db.schema": schema, + [`${label}.version`]: dbVersion, + [`${label}.schema_version`]: schemaVersion, + }); + return; + } + */ + + const incrementals = [...cfg.incrementals].sort((a, b) => + a.name.localeCompare(b.name), + ); + for (const migration of incrementals) { + const [existingRow] = await tx` + select exists ( + select 1 from ${tx(schema)}.migration where name = ${migration.name} + ) as existing + `; + if (existingRow?.existing) continue; + + const attributes = migrationAttributes( + label, + schema, + schemaVersion, + migration, + "incremental", + ); + await span(`${label}.migrate.incremental`, { + attributes, + callback: async () => { + await executeSqlFile(tx, template(migration.sql, templateVars), { + logSqlFiles, + label, + schema, + type: "incremental", + dir, + file: migration.file, + }); + await tx` + insert into ${tx(schema)}.migration (name, applied_at_version) + values (${migration.name}, ${schemaVersion})`; + }, + }); + info(`${Label} migration applied`, attributes); + } + + const idempotents = [...cfg.idempotents].sort((a, b) => + a.name.localeCompare(b.name), + ); + for (const migration of idempotents) { + await span(`${label}.migrate.idempotent`, { + attributes: migrationAttributes( + label, + schema, + schemaVersion, + migration, + "idempotent", + ), + callback: () => + executeSqlFile(tx, template(migration.sql, templateVars), { + logSqlFiles, + label, + schema, + type: "idempotent", + dir, + file: migration.file, + }), + }); + } + + await tx`update ${tx(schema)}.version set version = ${schemaVersion}, at = now()`; +} diff --git a/packages/database/space/migrate/bootstrap.ts b/packages/database/space/migrate/bootstrap.ts index 6a52e71..7721cbd 100644 --- a/packages/database/space/migrate/bootstrap.ts +++ b/packages/database/space/migrate/bootstrap.ts @@ -1,13 +1,13 @@ import { info, reportError, span } from "@pydantic/logfire-node"; -import { semver } from "bun"; -import type { ISql, Sql as SQL } from "postgres"; - -const REQUIRED_EXTENSIONS = [ - { name: "citext", minVersion: "1.6" }, - { name: "ltree", minVersion: "1.3" }, - { name: "vector", minVersion: "0.8.2" }, - { name: "pg_textsearch", minVersion: "1.1.0" }, -] as const; +import type { Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + ensurePostgresVersion, + ensureRequiredExtensions, + REQUIRED_EXTENSIONS, +} from "../../migrate/kit"; /** * Prepare a physical database to host space schemas. @@ -38,28 +38,25 @@ export async function bootstrapSpaceDatabase( attributes, callback: async () => { try { + const [key1, key2] = advisoryLockKey("memory-space:bootstrap"); await sql.begin(async (tx) => { if (shardId !== undefined) { await tx.unsafe(`set local pgdog.shard to ${String(shardId)}`); } await ensurePostgresVersion(tx); - await span("space.bootstrap.acquire_lock", { - callback: () => acquireAdvisoryLock(tx), + const acquired = await span("space.bootstrap.acquire_lock", { + callback: () => acquireAdvisoryLock(tx, key1, key2), }); - await tx`select set_config('statement_timeout', ${statementTimeout}, true)`; - await tx`select set_config('lock_timeout', ${lockTimeout}, true)`; - await tx`select set_config('transaction_timeout', ${transactionTimeout}, true)`; - await tx`select set_config('idle_in_transaction_session_timeout', ${idleInTransactionSessionTimeout}, true)`; - for (const extension of REQUIRED_EXTENSIONS) { - await span("space.bootstrap.ensure_extension", { - attributes: { - "db.extension": extension.name, - "db.extension_min_version": extension.minVersion, - }, - callback: () => - ensureExtension(tx, extension.name, extension.minVersion), - }); + if (!acquired) { + throw new Error("Failed to acquire advisory lock"); } + await applySessionTimeouts(tx, { + statementTimeout, + lockTimeout, + transactionTimeout, + idleInTransactionSessionTimeout, + }); + await ensureRequiredExtensions(tx, "space.bootstrap"); }); info("Space bootstrap completed", attributes); } catch (error) { @@ -69,85 +66,3 @@ export async function bootstrapSpaceDatabase( }, }); } - -const MAX_LOCK_RETRIES = 5; -const BASE_DELAY_MS = 100; -const BOOTSTRAP_LOCK_ID = 1982010637711; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function acquireAdvisoryLock(tx: ISql): Promise { - let acquired = false; - for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { - const [result] = await tx` - select pg_try_advisory_xact_lock(${BOOTSTRAP_LOCK_ID}) as acquired - `; - if (result?.acquired) { - acquired = true; - break; - } - if (attempt < MAX_LOCK_RETRIES - 1) { - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } - - if (!acquired) { - throw new Error(`Failed to acquire advisory lock`); - } -} - -async function ensurePostgresVersion(tx: ISql): Promise { - const [row] = await tx` - select current_setting('server_version_num')::int as server_version_num - `; - const serverVersionNum = Number(row?.server_version_num); - if (serverVersionNum < 180000) { - throw new Error( - `PostgreSQL version 18 or higher is required (found ${serverVersionNum})`, - ); - } -} - -async function ensureExtension( - tx: ISql, - name: string, - minVersion: string, -): Promise { - const [installed] = await tx` - select x.extversion, n.nspname - from pg_extension x - inner join pg_namespace n on (x.extnamespace = n.oid) - where x.extname = ${name} - `; - - if (installed) { - if ( - installed.nspname === "public" && - semver.order(installed.extversion, minVersion) >= 0 - ) { - return; - } - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required in the "public" schema (found ${installed.extversion} installed in "${installed.nspname}")`, - ); - } - - const [available] = await tx` - select default_version - from pg_available_extensions - where name = ${name} - `; - - if (!available || semver.order(available.default_version, minVersion) < 0) { - const found = available - ? `found ${available.default_version} available` - : "not available"; - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required (${found})`, - ); - } - - await tx`create extension if not exists ${tx(name)} with schema public`; -} diff --git a/packages/database/space/migrate/migrate.ts b/packages/database/space/migrate/migrate.ts index fb5beee..214df0a 100644 --- a/packages/database/space/migrate/migrate.ts +++ b/packages/database/space/migrate/migrate.ts @@ -1,22 +1,33 @@ -import { createHash } from "node:crypto"; import { info, reportError, span } from "@pydantic/logfire-node"; import { semver } from "bun"; -import type { ISql, Sql as SQL } from "postgres"; +import type { Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + doesSchemaExist, + executeSqlFile, + isValidSchemaName, + type Migration, + runSchemaMigrations, + template, +} from "../../migrate/kit"; import { isValidSlug, slugToSchema } from "../slug"; import { SPACE_SCHEMA_VERSION } from "../version"; +import idempotent001 from "./idempotent/001_memory.sql" with { type: "text" }; +import idempotent002 from "./idempotent/002_search.sql" with { type: "text" }; +import idempotent003 from "./idempotent/003_embedding_queue.sql" with { + type: "text", +}; import incremental001 from "./incremental/001_memory.sql" with { type: "text" }; import incremental002 from "./incremental/002_embedding_queue.sql" with { type: "text", }; import provisionSql from "./provision.sql" with { type: "text" }; -interface Incremental { - name: string; - file: string; - sql: string; -} +const DIR = "packages/database/space/migrate"; -const incrementals: Incremental[] = [ +const incrementals: Migration[] = [ { name: "001_memory", file: "incremental/001_memory.sql", @@ -29,19 +40,7 @@ const incrementals: Incremental[] = [ }, ]; -import idempotent001 from "./idempotent/001_memory.sql" with { type: "text" }; -import idempotent002 from "./idempotent/002_search.sql" with { type: "text" }; -import idempotent003 from "./idempotent/003_embedding_queue.sql" with { - type: "text", -}; - -interface Idempotent { - name: string; - file: string; - sql: string; -} - -const idempotents: Idempotent[] = [ +const idempotents: Migration[] = [ { name: "001_memory", file: "idempotent/001_memory.sql", sql: idempotent001 }, { name: "002_search", file: "idempotent/002_search.sql", sql: idempotent002 }, { @@ -130,10 +129,7 @@ export async function migrateSpace( } await tx.unsafe(`set local pgdog.shard to ${String(opts.shardId)}`); } - await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; - await tx`select set_config('lock_timeout', ${opts.lockTimeout}, true)`; - await tx`select set_config('transaction_timeout', ${opts.transactionTimeout}, true)`; - await tx`select set_config('idle_in_transaction_session_timeout', ${opts.idleInTransactionSessionTimeout}, true)`; + await applySessionTimeouts(tx, opts); const acquired = await span("space.migrate.acquire_lock", { attributes: schemaAttributes, callback: () => acquireAdvisoryLock(tx, key1, key2), @@ -144,20 +140,38 @@ export async function migrateSpace( ); } - if (!(await doesSpaceExist(tx, schema))) { + if (!(await doesSchemaExist(tx, schema))) { await span("space.migrate.provision", { attributes: { ...schemaAttributes, "space.migration_file": "provision.sql", "space.migration_type": "provision", }, - callback: () => provisionSpace(tx, schema, opts), + callback: () => + executeSqlFile(tx, template(provisionSql, { schema }), { + logSqlFiles: opts.logSqlFiles, + label: "space", + schema, + type: "provision", + dir: DIR, + file: "provision.sql", + }), }); info("Space schema provisioned", schemaAttributes); } await span("space.migrate.run", { attributes: schemaAttributes, - callback: () => runMigrations(tx, schema, opts), + callback: () => + runSchemaMigrations(tx, { + schema, + schemaVersion: opts.schemaVersion, + incrementals, + idempotents, + templateVars: templateVars(schema, opts), + label: "space", + dir: DIR, + logSqlFiles: opts.logSqlFiles, + }), }); }); info("Space migrations completed", schemaAttributes); @@ -222,304 +236,3 @@ function templateVars( hnsw_ef_construction: options.hnswEfConstruction, }; } - -function isValidSchemaName(schema: string): boolean { - return /^[a-z_][a-z0-9_]*$/.test(schema) && schema.length <= 63; -} - -function advisoryLockKey(schema: string): [number, number] { - const digest = createHash("sha256").update(schema).digest(); - return [digest.readInt32BE(0), digest.readInt32BE(4)]; -} - -const MAX_LOCK_RETRIES = 5; -const BASE_DELAY_MS = 100; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function acquireAdvisoryLock( - tx: ISql, - key1: number, - key2: number, -): Promise { - let acquired = false; - for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { - const [result] = await tx` - select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired - `; - if (result?.acquired) { - acquired = true; - break; - } - if (attempt < MAX_LOCK_RETRIES - 1) { - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } - return acquired; -} - -async function doesSpaceExist(tx: ISql, schema: string): Promise { - const [row] = await tx` - select exists - ( - select 1 - from pg_namespace n - where n.nspname = ${schema} - ) as "spaceExists" - `; - return Boolean(row?.spaceExists); -} - -async function provisionSpace( - tx: ISql, - schema: string, - options: NormalizedMigrateSpaceOptions, -): Promise { - await executeSqlFile( - tx, - options, - schema, - "provision", - "provision.sql", - template(provisionSql, { schema }), - ); -} - -async function runMigrations( - tx: ISql, - schema: string, - options: NormalizedMigrateSpaceOptions, -): Promise { - // check ownership - await assertSchemaOwnership(tx, schema); - - // check version - const [versionRow] = await tx` - select version from ${tx(schema)}.version - `; - const dbVersion: string = versionRow?.version; - const cmp = semver.order(options.schemaVersion, dbVersion); - // abort if target is older than the database - if (cmp < 0) { - throw new Error( - `Application version (${options.schemaVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the application.", - ); - } - /* run migrations regardless - if (cmp === 0) { - // version matches. no need to run migrations - info("Space migration skipped, version current", { - "db.schema": schema, - "space.version": dbVersion, - "space.schema_version": options.schemaVersion, - }); - return; - } - */ - - // run incremental migrations - const sorted1 = [...incrementals].sort((a, b) => - a.name.localeCompare(b.name), - ); - - for (const migration of sorted1) { - const [existingRow] = await tx` - select exists - ( - select 1 - from ${tx(schema)}.migration - where name = ${migration.name} - ) as existing - `; - - if (existingRow?.existing) { - continue; - } - - await span("space.migrate.incremental", { - attributes: { - "db.schema": schema, - "space.migration": migration.name, - "space.migration_file": migration.file, - "space.migration_type": "incremental", - "space.schema_version": options.schemaVersion, - }, - callback: async () => { - const renderedSql = template( - migration.sql, - templateVars(schema, options), - ); - await executeSqlFile( - tx, - options, - schema, - "incremental", - migration.file, - renderedSql, - ); - await tx` - insert into ${tx(schema)}.migration (name, applied_at_version) - values (${migration.name}, ${options.schemaVersion})`; - }, - }); - info("Space migration applied", { - "db.schema": schema, - "space.migration": migration.name, - "space.migration_file": migration.file, - "space.migration_type": "incremental", - "space.schema_version": options.schemaVersion, - }); - } - - // run idempotent migrations - const sorted2 = [...idempotents].sort((a, b) => a.name.localeCompare(b.name)); - - for (const migration of sorted2) { - await span("space.migrate.idempotent", { - attributes: { - "db.schema": schema, - "space.migration": migration.name, - "space.migration_file": migration.file, - "space.migration_type": "idempotent", - "space.schema_version": options.schemaVersion, - }, - callback: async () => { - const renderedSql = template( - migration.sql, - templateVars(schema, options), - ); - await executeSqlFile( - tx, - options, - schema, - "idempotent", - migration.file, - renderedSql, - ); - }, - }); - } - - // update version - await tx`update ${tx(schema)}.version set version = ${options.schemaVersion}, at = now()`; -} - -async function executeSqlFile( - tx: ISql, - options: NormalizedMigrateSpaceOptions, - schema: string, - type: string, - file: string, - sqlText: string, -): Promise { - logSqlFile(options, schema, type, file); - try { - await tx.unsafe(sqlText); - } catch (error) { - logSqlExecutionError(options, schema, type, file, sqlText, error); - throw error; - } -} - -function logSqlFile( - options: NormalizedMigrateSpaceOptions, - schema: string, - type: string, - file: string, -): void { - if (!options.logSqlFiles) return; - console.error( - `[migrate:db] space ${schema} ${type} packages/space/migrate/${file}`, - ); -} - -function logSqlExecutionError( - options: NormalizedMigrateSpaceOptions, - schema: string, - type: string, - file: string, - sqlText: string, - error: unknown, -): void { - if (!options.logSqlFiles) return; - console.error( - `[migrate:db] failed space ${schema} ${type} packages/space/migrate/${file}`, - ); - logPostgresSqlLocation(sqlText, error); -} - -function logPostgresSqlLocation(sqlText: string, error: unknown): void { - // postgres-js sets `position` (1-based) on server errors; non-PG errors won't. - const position = Number((error as { position?: unknown })?.position); - if (!Number.isSafeInteger(position) || position < 1) return; - - const location = sqlLocation(sqlText, position); - if (!location) return; - console.error( - `[migrate:db] sql position ${position} -> line ${location.line}, column ${location.column}`, - ); - console.error(sqlContext(sqlText, location.line, location.column)); -} - -function sqlLocation( - sqlText: string, - position: number, -): { line: number; column: number } | undefined { - if (position > sqlText.length + 1) return undefined; - let line = 1; - let column = 1; - for (let i = 0; i < position - 1; i++) { - if (sqlText.charCodeAt(i) === 10) { - line++; - column = 1; - } else { - column++; - } - } - return { line, column }; -} - -function sqlContext(sqlText: string, line: number, column: number): string { - const lines = sqlText.split("\n"); - const start = Math.max(1, line - 2); - const end = Math.min(lines.length, line + 2); - const width = String(end).length; - const output = ["[migrate:db] sql context:"]; - - for (let n = start; n <= end; n++) { - const marker = n === line ? ">" : " "; - output.push(`${marker} ${String(n).padStart(width)} | ${lines[n - 1]}`); - if (n === line) { - output.push(` ${" ".repeat(width)} | ${" ".repeat(column - 1)}^`); - } - } - - return output.join("\n"); -} - -async function assertSchemaOwnership(tx: ISql, schema: string): Promise { - const [result] = await tx` - select - n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner - from pg_catalog.pg_namespace n - where n.nspname = ${schema} - `; - - if (!result?.is_owner) { - throw new Error( - `Only the owner of the ${schema} schema can run database migrations`, - ); - } -} - -function template(sql: string, vars: Record): string { - return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { - if (!(key in vars)) { - throw new Error(`Missing template variable: ${key}`); - } - return String(vars[key]); - }); -} From 307160327baef0312bfb71dab6cd0559b21a3072 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 2 Jun 2026 20:38:10 +0200 Subject: [PATCH 025/156] feat(database): add auth schema migration unit (better-auth-shaped) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New packages/database/auth migration unit, a sibling to core/ and space/ using the shared kit.ts runner. Provisions an `auth` schema with better-auth-shaped tables: users, accounts, sessions, device_authorization, and verifications. - Login-only OAuth: the accounts token/password columns are kept for better-auth shape parity but left nullable and never written, so there is no token-encryption subsystem. - sessions store token_hash (sha256), not a raw token — a deliberate divergence from better-auth's plaintext lookup so a DB read yields no usable bearer tokens. - Requires only citext (not the engine ltree/vector/pg_textsearch extensions), so the schema can live in a pgvector-less database. Verified: typecheck, lint, and 20/20 integration tests against local PG18. Against remote testing_me the connection-storm tests time out the same way the existing core suite does (a shared, pre-existing latency issue, not this unit). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/database/auth/index.ts | 6 + .../auth/migrate/idempotent/000_update.sql | 27 ++ .../database/auth/migrate/idempotent/sql.d.ts | 4 + .../auth/migrate/incremental/001_users.sql | 13 + .../auth/migrate/incremental/002_accounts.sql | 26 ++ .../auth/migrate/incremental/003_sessions.sql | 22 ++ .../incremental/004_device_authorization.sql | 19 ++ .../migrate/incremental/005_verifications.sql | 17 + .../auth/migrate/incremental/sql.d.ts | 4 + .../auth/migrate/migrate.integration.test.ts | 322 ++++++++++++++++++ packages/database/auth/migrate/migrate.ts | 220 ++++++++++++ packages/database/auth/migrate/provision.sql | 15 + packages/database/auth/migrate/sql.d.ts | 4 + packages/database/auth/migrate/test-utils.ts | 65 ++++ packages/database/auth/version.ts | 1 + packages/database/index.ts | 4 +- 16 files changed, 768 insertions(+), 1 deletion(-) create mode 100644 packages/database/auth/index.ts create mode 100644 packages/database/auth/migrate/idempotent/000_update.sql create mode 100644 packages/database/auth/migrate/idempotent/sql.d.ts create mode 100644 packages/database/auth/migrate/incremental/001_users.sql create mode 100644 packages/database/auth/migrate/incremental/002_accounts.sql create mode 100644 packages/database/auth/migrate/incremental/003_sessions.sql create mode 100644 packages/database/auth/migrate/incremental/004_device_authorization.sql create mode 100644 packages/database/auth/migrate/incremental/005_verifications.sql create mode 100644 packages/database/auth/migrate/incremental/sql.d.ts create mode 100644 packages/database/auth/migrate/migrate.integration.test.ts create mode 100644 packages/database/auth/migrate/migrate.ts create mode 100644 packages/database/auth/migrate/provision.sql create mode 100644 packages/database/auth/migrate/sql.d.ts create mode 100644 packages/database/auth/migrate/test-utils.ts create mode 100644 packages/database/auth/version.ts diff --git a/packages/database/auth/index.ts b/packages/database/auth/index.ts new file mode 100644 index 0000000..8e5c50d --- /dev/null +++ b/packages/database/auth/index.ts @@ -0,0 +1,6 @@ +export { + AUTH_SCHEMA, + type MigrateAuthOptions, + migrateAuth, +} from "./migrate/migrate"; +export { AUTH_SCHEMA_VERSION } from "./version"; diff --git a/packages/database/auth/migrate/idempotent/000_update.sql b/packages/database/auth/migrate/idempotent/000_update.sql new file mode 100644 index 0000000..78e9f07 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/000_update.sql @@ -0,0 +1,27 @@ +-- generic trigger function to update updated_at timestamp +create or replace function {{schema}}.update_updated_at() +returns trigger +as $func$ +begin + new.updated_at = pg_catalog.now(); + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, pg_temp; + +-- only tables that carry an updated_at column get the trigger +-- (sessions and device_authorization are insert/delete-only and have none) +create or replace trigger users_before_update_trg +before update on {{schema}}.users +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger accounts_before_update_trg +before update on {{schema}}.accounts +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger verifications_before_update_trg +before update on {{schema}}.verifications +for each row +execute function {{schema}}.update_updated_at(); diff --git a/packages/database/auth/migrate/idempotent/sql.d.ts b/packages/database/auth/migrate/idempotent/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/auth/migrate/incremental/001_users.sql b/packages/database/auth/migrate/incremental/001_users.sql new file mode 100644 index 0000000..2c8df93 --- /dev/null +++ b/packages/database/auth/migrate/incremental/001_users.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- users (better-auth model: user) +-- "users" (plural) avoids the SQL reserved word "user". +------------------------------------------------------------------------------- +create table {{schema}}.users +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, name text not null +, email citext not null unique -- citext: case-insensitive, even if app-layer lowercasing is bypassed +, email_verified boolean not null default false -- set from the provider's verified-email flag +, image text -- optional avatar url (better-auth parity) +, created_at timestamptz not null default now() +, updated_at timestamptz -- maintained by update_updated_at() trigger (idempotent/000) +); diff --git a/packages/database/auth/migrate/incremental/002_accounts.sql b/packages/database/auth/migrate/incremental/002_accounts.sql new file mode 100644 index 0000000..aebae98 --- /dev/null +++ b/packages/database/auth/migrate/incremental/002_accounts.sql @@ -0,0 +1,26 @@ +------------------------------------------------------------------------------- +-- accounts (better-auth model: account) +-- One row per provider link. LOGIN-ONLY: we authenticate via GitHub/Google but +-- never call their APIs on the user's behalf, so the token/password columns are +-- kept (for better-auth shape parity) but left null and never written. Because +-- nothing sensitive is stored at rest, there is no token-encryption subsystem. +------------------------------------------------------------------------------- +create table {{schema}}.accounts +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid not null references {{schema}}.users (id) on delete cascade +, provider_id text not null check (provider_id in ('google', 'github')) -- was `provider` +, account_id text not null -- provider's stable user id (was `provider_account_id`) +, access_token text -- nullable, unused (login-only) +, refresh_token text -- nullable, unused +, id_token text -- nullable, unused +, access_token_expires_at timestamptz +, refresh_token_expires_at timestamptz +, scope text +, password text -- nullable, unused (OAuth-only, no email/password) +, created_at timestamptz not null default now() +, updated_at timestamptz +-- the OAuth sign-in lookup key + integrity rule: one external account -> one row +, unique (provider_id, account_id) +); + +create index accounts_user_id_idx on {{schema}}.accounts (user_id); diff --git a/packages/database/auth/migrate/incremental/003_sessions.sql b/packages/database/auth/migrate/incremental/003_sessions.sql new file mode 100644 index 0000000..19ad415 --- /dev/null +++ b/packages/database/auth/migrate/incremental/003_sessions.sql @@ -0,0 +1,22 @@ +------------------------------------------------------------------------------- +-- sessions (better-auth model: session) +-- DELIBERATE DIVERGENCE FROM better-auth: we store sha256(token) in `token_hash`, +-- not the raw token. Our own validateSession hashes the presented token and looks +-- up by hash, so a database read never yields usable bearer tokens. (The BA library +-- reads sessions by raw-token equality and can't hash here; if we ever adopt it, +-- reconciling is cheap — switch to a plaintext `token` column and truncate, since +-- sessions are disposable.) +------------------------------------------------------------------------------- +create table {{schema}}.sessions +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid not null references {{schema}}.users (id) on delete cascade +, token_hash bytea not null -- sha256(rawToken); rawToken is 256-bit CSPRNG, shown to client only +, expires_at timestamptz not null +, ip_address text -- better-auth parity, nullable +, user_agent text -- better-auth parity, nullable +, created_at timestamptz not null default now() +); + +create unique index sessions_token_hash_uniq on {{schema}}.sessions (token_hash); -- the auth lookup +create index sessions_user_id_idx on {{schema}}.sessions (user_id); -- revoke-all-by-user +create index sessions_expires_at_idx on {{schema}}.sessions (expires_at); -- expired-row sweeps diff --git a/packages/database/auth/migrate/incremental/004_device_authorization.sql b/packages/database/auth/migrate/incremental/004_device_authorization.sql new file mode 100644 index 0000000..2754563 --- /dev/null +++ b/packages/database/auth/migrate/incremental/004_device_authorization.sql @@ -0,0 +1,19 @@ +------------------------------------------------------------------------------- +-- device_authorization (OAuth 2.0 device flow — RFC 8628) +-- Our own device-flow state (not a better-auth table). `user_id` (was identity_id) +-- is filled in by the OAuth callback once the human authorizes; the CLI polls by +-- device_code and exchanges an authorized row for a session. +------------------------------------------------------------------------------- +create table {{schema}}.device_authorization +( device_code text not null primary key -- CLI polling secret (32-byte base64url) +, user_code text not null unique -- human-entered code, XXXX-XXXX +, provider text not null check (provider in ('google', 'github')) +, oauth_state text not null unique -- CSRF binding for the OAuth callback +, expires_at timestamptz not null -- short TTL (~15 min) +, last_poll timestamptz -- rate-limiting the CLI poll +, user_id uuid references {{schema}}.users (id) on delete cascade -- null until authorized +, denied boolean not null default false +, created_at timestamptz not null default now() +); + +create index device_authorization_expires_at_idx on {{schema}}.device_authorization (expires_at); -- expired-row sweeps diff --git a/packages/database/auth/migrate/incremental/005_verifications.sql b/packages/database/auth/migrate/incremental/005_verifications.sql new file mode 100644 index 0000000..dbf1e5b --- /dev/null +++ b/packages/database/auth/migrate/incremental/005_verifications.sql @@ -0,0 +1,17 @@ +------------------------------------------------------------------------------- +-- verifications (better-auth model: verification) +-- Generic key/value-with-expiry store: email verification, password reset, magic +-- links, OTPs. Unused today (we do GitHub/Google OAuth only) but kept empty for +-- better-auth shape parity, so enabling the library later needs no migration. +------------------------------------------------------------------------------- +create table {{schema}}.verifications +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, identifier text not null +, value text not null +, expires_at timestamptz not null +, created_at timestamptz not null default now() +, updated_at timestamptz +); + +create index verifications_identifier_idx on {{schema}}.verifications (identifier); +create index verifications_expires_at_idx on {{schema}}.verifications (expires_at); -- expired-row sweeps diff --git a/packages/database/auth/migrate/incremental/sql.d.ts b/packages/database/auth/migrate/incremental/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/auth/migrate/incremental/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/auth/migrate/migrate.integration.test.ts b/packages/database/auth/migrate/migrate.integration.test.ts new file mode 100644 index 0000000..c603d36 --- /dev/null +++ b/packages/database/auth/migrate/migrate.integration.test.ts @@ -0,0 +1,322 @@ +// Integration tests for the `auth` schema migrations (migrateAuth). +// +// The auth migrations are templated, so each test targets its own throwaway +// `auth_test_` schema — never the real `auth`. That makes these tests +// isolated and safe to run against any database (including a shared dev one). +// Read-only shape assertions share one canonical auth schema provisioned in +// beforeAll; the few behavior tests provision their own. +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Sql as SQL } from "postgres"; +import { AUTH_SCHEMA_VERSION } from "../version"; +import { migrateAuth } from "./migrate"; +import { + appliedMigrations, + connect, + expectReject, + extensionInstalled, + getSchemaVersion, + listFunctions, + listTables, + listTriggers, + randomAuthSchema, + schemaExists, + TestAuth, + tableExists, + withTestAuth, +} from "./test-utils"; + +const EXPECTED_TABLES = [ + "accounts", + "device_authorization", + "migration", + "sessions", + "users", + "verifications", + "version", +]; + +const EXPECTED_MIGRATIONS = [ + "001_users", + "002_accounts", + "003_sessions", + "004_device_authorization", + "005_verifications", +]; + +const EXPECTED_FUNCTIONS = ["update_updated_at"]; + +// The auth schema deliberately requires only citext — not the engine extensions. +const REQUIRED_EXTENSIONS = ["citext"]; + +const V7 = "00000000-0000-7000-8000-000000000000"; +const V4 = "00000000-0000-4000-8000-000000000000"; + +/** Insert a user and return its id (most tables FK to users). */ +async function insertUser(sql: SQL, schema: string): Promise { + const email = `u_${crypto.randomUUID().slice(0, 8)}@example.com`; + const [row] = await sql.unsafe( + `insert into ${schema}.users (name, email) values ('Test', '${email}') returning id`, + ); + return row?.id as string; +} + +let sql: SQL; +// One migrated auth schema shared by all read-only shape/function assertions. +let canonical: TestAuth; + +beforeAll(async () => { + sql = connect(12); + canonical = await TestAuth.create(sql); // migrateAuth installs citext itself +}); + +afterAll(async () => { + await canonical?.drop(); + await sql.end(); +}); + +describe("provisioned auth schema", () => { + test("provisions into the requested (templated) schema", async () => { + expect(canonical.schema).toMatch(/^auth_test_/); + expect(await schemaExists(sql, canonical.schema)).toBe(true); + }); + + test("creates infrastructure and domain tables", async () => { + const tables = await listTables(sql, canonical.schema); + for (const table of EXPECTED_TABLES) { + expect(tables).toContain(table); + } + }); + + test("records every incremental migration exactly once", async () => { + expect(await appliedMigrations(sql, canonical.schema)).toEqual( + EXPECTED_MIGRATIONS, + ); + }); + + test("stamps the schema version", async () => { + expect(await getSchemaVersion(sql, canonical.schema)).toBe( + AUTH_SCHEMA_VERSION, + ); + }); + + test("installs only the required extensions", async () => { + for (const ext of REQUIRED_EXTENSIONS) { + expect(await extensionInstalled(sql, ext)).toBe(true); + } + }); + + test("creates the updated_at trigger function in the schema", async () => { + const functions = await listFunctions(sql, canonical.schema); + for (const fn of EXPECTED_FUNCTIONS) { + expect(functions).toContain(fn); + } + }); + + test("installs updated_at triggers on mutable tables only", async () => { + for (const table of ["users", "accounts", "verifications"]) { + const triggers = await listTriggers(sql, canonical.schema, table); + expect(triggers).toContain(`${table}_before_update_trg`); + } + // insert/delete-only tables have no updated_at and thus no trigger + for (const table of ["sessions", "device_authorization"]) { + const triggers = await listTriggers(sql, canonical.schema, table); + expect(triggers).not.toContain(`${table}_before_update_trg`); + } + }); +}); + +describe("schema constraints enforce", () => { + test("user ids must be UUIDv7", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.users (id, name, email) + values ('${V4}', 'v4', 'v4@example.com')`, + ), + ); + }); + + test("user email is unique and case-insensitive (citext)", async () => { + const s = canonical.schema; + const email = `Dup_${crypto.randomUUID().slice(0, 8)}@Example.com`; + await sql.unsafe( + `insert into ${s}.users (name, email) values ('a', '${email}')`, + ); + try { + await expectReject(() => + sql.unsafe( + `insert into ${s}.users (name, email) values ('b', '${email.toLowerCase()}')`, + ), + ); + } finally { + await sql.unsafe(`delete from ${s}.users where email = '${email}'`); + } + }); + + test("accounts.provider_id is restricted to google/github", async () => { + const userId = await insertUser(sql, canonical.schema); + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.accounts (user_id, provider_id, account_id) + values ('${userId}', 'facebook', 'x')`, + ), + ); + }); + + test("accounts are unique per (provider_id, account_id)", async () => { + const s = canonical.schema; + const userId = await insertUser(sql, s); + const acct = crypto.randomUUID(); + await sql.unsafe( + `insert into ${s}.accounts (user_id, provider_id, account_id) + values ('${userId}', 'github', '${acct}')`, + ); + await expectReject(() => + sql.unsafe( + `insert into ${s}.accounts (user_id, provider_id, account_id) + values ('${userId}', 'github', '${acct}')`, + ), + ); + }); + + test("accounts.user_id must reference an existing user", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.accounts (user_id, provider_id, account_id) + values ('${V7}', 'github', 'orphan')`, + ), + ); + }); + + test("session token_hash is unique", async () => { + const s = canonical.schema; + const userId = await insertUser(sql, s); + await sql.unsafe( + `insert into ${s}.sessions (user_id, token_hash, expires_at) + values ('${userId}', '\\xdeadbeef', now() + interval '1 day')`, + ); + await expectReject(() => + sql.unsafe( + `insert into ${s}.sessions (user_id, token_hash, expires_at) + values ('${userId}', '\\xdeadbeef', now() + interval '1 day')`, + ), + ); + }); + + test("device_authorization.user_code is unique", async () => { + const s = canonical.schema; + const code = `AB${crypto.randomUUID().slice(0, 4).toUpperCase()}`; + await sql.unsafe( + `insert into ${s}.device_authorization (device_code, user_code, provider, oauth_state, expires_at) + values ('${crypto.randomUUID()}', '${code}', 'google', '${crypto.randomUUID()}', now() + interval '15 min')`, + ); + await expectReject(() => + sql.unsafe( + `insert into ${s}.device_authorization (device_code, user_code, provider, oauth_state, expires_at) + values ('${crypto.randomUUID()}', '${code}', 'google', '${crypto.randomUUID()}', now() + interval '15 min')`, + ), + ); + }); +}); + +describe("cascade + trigger behavior", () => { + test("deleting a user cascades to accounts and sessions", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const userId = await insertUser(sql, s); + await sql.unsafe( + `insert into ${s}.accounts (user_id, provider_id, account_id) values ('${userId}', 'github', '1')`, + ); + await sql.unsafe( + `insert into ${s}.sessions (user_id, token_hash, expires_at) values ('${userId}', '\\xaa', now() + interval '1 day')`, + ); + + await sql.unsafe(`delete from ${s}.users where id = '${userId}'`); + + const [acct] = await sql.unsafe( + `select count(*)::int as n from ${s}.accounts where user_id = '${userId}'`, + ); + const [sess] = await sql.unsafe( + `select count(*)::int as n from ${s}.sessions where user_id = '${userId}'`, + ); + expect(acct?.n).toBe(0); + expect(sess?.n).toBe(0); + }); + }); + + test("updating a user bumps updated_at via trigger", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const userId = await insertUser(sql, s); + const [before] = await sql.unsafe( + `select updated_at from ${s}.users where id = '${userId}'`, + ); + expect(before?.updated_at).toBeNull(); + + await sql.unsafe( + `update ${s}.users set name = 'Renamed' where id = '${userId}'`, + ); + const [after] = await sql.unsafe( + `select updated_at from ${s}.users where id = '${userId}'`, + ); + expect(after?.updated_at).not.toBeNull(); + }); + }); +}); + +describe("migration behavior", () => { + test("is idempotent: re-running changes no migration rows or version", async () => { + await withTestAuth(sql, {}, async (auth) => { + const before = await appliedMigrations(sql, auth.schema); + await migrateAuth(sql, { schema: auth.schema }); + expect(await appliedMigrations(sql, auth.schema)).toEqual(before); + expect(await getSchemaVersion(sql, auth.schema)).toBe( + AUTH_SCHEMA_VERSION, + ); + }); + }); + + test("rejects a downgrade (db version newer than app)", async () => { + await withTestAuth(sql, {}, async (auth) => { + await sql.unsafe(`update ${auth.schema}.version set version = '99.0.0'`); + await expect(migrateAuth(sql, { schema: auth.schema })).rejects.toThrow( + /older than database version/, + ); + }); + }); + + test("rejects invalid schema names", async () => { + for (const schema of ["Bad-Schema", "1auth", "auth test", "auth;drop"]) { + await expect(migrateAuth(sql, { schema })).rejects.toThrow( + /Invalid auth schema name/, + ); + } + }); + + test("concurrent migrateAuth on one schema is serialized safely", async () => { + // The advisory lock serializes writers. A loser may exhaust its retry + // budget and throw "Unable to acquire lock" — expected, not corruption. + // What must hold: at least one succeeds and the schema stays valid. + const schema = randomAuthSchema(); + try { + const results = await Promise.allSettled([ + migrateAuth(sql, { schema }), + migrateAuth(sql, { schema }), + migrateAuth(sql, { schema }), + ]); + + expect(results.some((r) => r.status === "fulfilled")).toBe(true); + for (const r of results) { + if (r.status === "rejected") { + expect(String((r.reason as Error)?.message ?? r.reason)).toContain( + "Unable to acquire lock", + ); + } + } + + expect(await getSchemaVersion(sql, schema)).toBe(AUTH_SCHEMA_VERSION); + expect(await tableExists(sql, schema, "users")).toBe(true); + } finally { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + } + }); +}); diff --git a/packages/database/auth/migrate/migrate.ts b/packages/database/auth/migrate/migrate.ts new file mode 100644 index 0000000..d3e4014 --- /dev/null +++ b/packages/database/auth/migrate/migrate.ts @@ -0,0 +1,220 @@ +import { info, reportError, span } from "@pydantic/logfire-node"; +import { semver } from "bun"; +import type { Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + doesSchemaExist, + ensureExtension, + ensurePostgresVersion, + executeSqlFile, + isValidSchemaName, + type Migration, + runSchemaMigrations, + template, +} from "../../migrate/kit"; +import { AUTH_SCHEMA_VERSION } from "../version"; +import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; +import incremental001 from "./incremental/001_users.sql" with { type: "text" }; +import incremental002 from "./incremental/002_accounts.sql" with { + type: "text", +}; +import incremental003 from "./incremental/003_sessions.sql" with { + type: "text", +}; +import incremental004 from "./incremental/004_device_authorization.sql" with { + type: "text", +}; +import incremental005 from "./incremental/005_verifications.sql" with { + type: "text", +}; +import provisionSql from "./provision.sql" with { type: "text" }; + +const DIR = "packages/database/auth/migrate"; + +// The auth schema only needs citext (case-insensitive email). It deliberately +// does NOT require the engine extensions (ltree / vector / pg_textsearch), so it +// can live in a database with no pgvector. +const AUTH_REQUIRED_EXTENSIONS = [ + { name: "citext", minVersion: "1.6" }, +] as const; + +const incrementals: Migration[] = [ + { name: "001_users", file: "incremental/001_users.sql", sql: incremental001 }, + { + name: "002_accounts", + file: "incremental/002_accounts.sql", + sql: incremental002, + }, + { + name: "003_sessions", + file: "incremental/003_sessions.sql", + sql: incremental003, + }, + { + name: "004_device_authorization", + file: "incremental/004_device_authorization.sql", + sql: incremental004, + }, + { + name: "005_verifications", + file: "incremental/005_verifications.sql", + sql: incremental005, + }, +]; + +const idempotents: Migration[] = [ + { name: "000_update", file: "idempotent/000_update.sql", sql: idempotent000 }, +]; + +/** + * The authentication schema name. Production uses "auth"; the name is a + * parameter so tests can provision throwaway, isolated auth schemas (and so the + * SQL is templated symmetrically with the core/space migrations). Reference this + * constant rather than hardcoding "auth" elsewhere. + */ +export const AUTH_SCHEMA = "auth"; + +export interface MigrateAuthOptions { + schema?: string; + logSqlFiles?: boolean; + statementTimeout?: string; + lockTimeout?: string; + transactionTimeout?: string; + idleInTransactionSessionTimeout?: string; +} + +interface NormalizedMigrateAuthOptions { + schema: string; + logSqlFiles: boolean; + schemaVersion: string; + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function migrateAuth( + sql: SQL, + options: MigrateAuthOptions = {}, +): Promise { + const opts = normalizeMigrateAuthOptions(options); + const attributes = migrateAttributes(opts); + + await span("auth.migrate", { + attributes, + callback: async () => { + try { + if (!isValidSchemaName(opts.schema)) { + throw new Error( + `Invalid auth schema name: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, + ); + } + if (!semver.satisfies(opts.schemaVersion, "*")) { + throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); + } + const [key1, key2] = advisoryLockKey( + `memory-auth:schema:${opts.schema}`, + ); + + await sql.begin(async (tx) => { + await applySessionTimeouts(tx, opts); + const acquired = await span("auth.migrate.acquire_lock", { + attributes, + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error("Unable to acquire lock for auth migrations."); + } + + await ensurePostgresVersion(tx); + for (const extension of AUTH_REQUIRED_EXTENSIONS) { + await span("auth.migrate.ensure_extension", { + attributes: { + "db.extension": extension.name, + "db.extension_min_version": extension.minVersion, + }, + callback: () => + ensureExtension(tx, extension.name, extension.minVersion), + }); + } + + if (!(await doesSchemaExist(tx, opts.schema))) { + await span("auth.migrate.provision", { + attributes: { + ...attributes, + "auth.migration_file": "provision.sql", + "auth.migration_type": "provision", + }, + callback: () => + executeSqlFile( + tx, + template(provisionSql, { schema: opts.schema }), + { + logSqlFiles: opts.logSqlFiles, + label: "auth", + schema: opts.schema, + type: "provision", + dir: DIR, + file: "provision.sql", + }, + ), + }); + info("Auth schema provisioned", attributes); + } + await span("auth.migrate.run", { + attributes, + callback: () => + runSchemaMigrations(tx, { + schema: opts.schema, + schemaVersion: opts.schemaVersion, + incrementals, + idempotents, + templateVars: { schema: opts.schema }, + label: "auth", + dir: DIR, + logSqlFiles: opts.logSqlFiles, + }), + }); + }); + info("Auth migrations completed", attributes); + } catch (error) { + reportError("Auth migration failed", error as Error, attributes); + throw error; + } + }, + }); +} + +function migrateAttributes( + options: NormalizedMigrateAuthOptions, +): Record { + return { + "db.schema": options.schema, + "auth.schema_version": options.schemaVersion, + "auth.required_extensions": AUTH_REQUIRED_EXTENSIONS.map( + (extension) => `${extension.name}@>=${extension.minVersion}`, + ), + "db.statement_timeout": options.statementTimeout, + "db.lock_timeout": options.lockTimeout, + "db.transaction_timeout": options.transactionTimeout, + "db.idle_in_transaction_session_timeout": + options.idleInTransactionSessionTimeout, + }; +} + +function normalizeMigrateAuthOptions( + options: MigrateAuthOptions, +): NormalizedMigrateAuthOptions { + return { + schema: options.schema ?? AUTH_SCHEMA, + logSqlFiles: options.logSqlFiles ?? false, + schemaVersion: AUTH_SCHEMA_VERSION, + statementTimeout: options.statementTimeout ?? "20s", + lockTimeout: options.lockTimeout ?? "5s", + transactionTimeout: options.transactionTimeout ?? "1min", + idleInTransactionSessionTimeout: + options.idleInTransactionSessionTimeout ?? "5s", + }; +} diff --git a/packages/database/auth/migrate/provision.sql b/packages/database/auth/migrate/provision.sql new file mode 100644 index 0000000..e98b9d9 --- /dev/null +++ b/packages/database/auth/migrate/provision.sql @@ -0,0 +1,15 @@ +create schema {{schema}}; + +create table {{schema}}.version +( version text not null +, at timestamptz not null default now() +); + +create unique index version_singleton_idx on {{schema}}.version ((true)); -- only ONE row allowed +insert into {{schema}}.version (version) values ('0.0.0'); + +create table {{schema}}.migration +( name text not null constraint migration_pkey primary key +, applied_at_version text not null +, applied_at timestamptz not null default pg_catalog.clock_timestamp() +); diff --git a/packages/database/auth/migrate/sql.d.ts b/packages/database/auth/migrate/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/auth/migrate/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/auth/migrate/test-utils.ts b/packages/database/auth/migrate/test-utils.ts new file mode 100644 index 0000000..2d242bb --- /dev/null +++ b/packages/database/auth/migrate/test-utils.ts @@ -0,0 +1,65 @@ +import type { Sql as SQL } from "postgres"; +import { type MigrateAuthOptions, migrateAuth } from "./migrate"; + +// Connection, failure assertions, and schema introspection are shared with the +// core and space suites. +export * from "../../migrate/test-utils"; + +const SCHEMA_SUFFIX_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; + +/** A unique, valid auth schema name, e.g. "auth_test_a1b2c3d4". */ +export function randomAuthSchema(): string { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let suffix = ""; + for (const b of bytes) suffix += SCHEMA_SUFFIX_ALPHABET[b % 36]; + return `auth_test_${suffix}`; +} + +// --------------------------------------------------------------------------- +// TestAuth — a provisioned, isolated auth schema +// --------------------------------------------------------------------------- +// +// The auth migrations are templated (production uses the "auth" schema; tests +// pass a unique throwaway name), so every test gets its own isolated auth schema +// and they run concurrently without ever touching a real `auth` schema. + +export class TestAuth { + readonly schema: string; + private readonly sql: SQL; + + private constructor(sql: SQL, schema: string) { + this.sql = sql; + this.schema = schema; + } + + static async create( + sql: SQL, + options: Omit & { schema?: string } = {}, + ): Promise { + const schema = options.schema ?? randomAuthSchema(); + await migrateAuth(sql, { ...options, schema }); + return new TestAuth(sql, schema); + } + + async drop(): Promise { + await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); + } +} + +/** + * Provision a fresh auth schema, run `fn` against it, and always drop it + * afterward. Safe to call from concurrent tests — each gets its own unique + * schema. + */ +export async function withTestAuth( + sql: SQL, + options: Omit & { schema?: string }, + fn: (auth: TestAuth) => Promise, +): Promise { + const auth = await TestAuth.create(sql, options); + try { + return await fn(auth); + } finally { + await auth.drop(); + } +} diff --git a/packages/database/auth/version.ts b/packages/database/auth/version.ts new file mode 100644 index 0000000..3bc3270 --- /dev/null +++ b/packages/database/auth/version.ts @@ -0,0 +1 @@ +export const AUTH_SCHEMA_VERSION = "0.0.1"; diff --git a/packages/database/index.ts b/packages/database/index.ts index 7b9a573..825a189 100644 --- a/packages/database/index.ts +++ b/packages/database/index.ts @@ -2,6 +2,8 @@ // live in one package; they are co-located in a single database/deployment. // Kept as separate `core/` and `space/` modules so the boundary stays clean // (space must not import core) and the split is easy to undo if spaces are ever -// distributed across databases again. +// distributed across databases again. The `auth` schema (better-auth-shaped +// users/sessions/accounts) is its own module for the same reason. +export * from "./auth"; export * from "./core"; export * from "./space"; From 8948d7c4e58ec9539b43c66da4fc442b43f800a5 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 2 Jun 2026 20:38:56 +0200 Subject: [PATCH 026/156] test(database): run test:db suites at -P 2, not -P 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With three integration suites now (core, space, auth), running all of them in parallel (-P 4) saturates the small ghost test instance — concurrent pools plus simultaneous HNSW/BM25 index builds peg it, surfacing as CONNECT_TIMEOUT and 30s statement timeouts. -P 2 keeps two suites in flight, within capacity. Verified: full test:db green on ghost (core 18, space 19, auth 20). Co-Authored-By: Claude Opus 4.8 (1M context) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 78b32ba..2d25bf9 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "server": "./bun run packages/server/index.ts", "setup": "./bun scripts/setup.ts", "test": "./bun test packages", - "test:db": "./bun run test:db:clean && find packages/database -name '*.integration.test.ts' -print0 | xargs -0 -P 4 -n 1 ./bun test --timeout 30000", + "test:db": "./bun run test:db:clean && find packages/database -name '*.integration.test.ts' -print0 | xargs -0 -P 2 -n 1 ./bun test --timeout 30000", "test:db:clean": "./bun scripts/clean-test-schemas.ts", "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", "typecheck": "tsc --noEmit" From 28ea068af2e28469785cd650c9c78015ac1f5687 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 2 Jun 2026 20:49:46 +0200 Subject: [PATCH 027/156] refactor(database): drop shard/pgdog routing from space migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spaces are co-located in one database (pgdog distribution is off the table for now), so migrateSpace and bootstrapSpaceDatabase no longer accept a shardId or emit `set local pgdog.shard`. Pure removal — no caller passed shardId, and the space suite stays green (19/19). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/database/space/migrate/bootstrap.ts | 5 ----- packages/database/space/migrate/migrate.ts | 12 ------------ 2 files changed, 17 deletions(-) diff --git a/packages/database/space/migrate/bootstrap.ts b/packages/database/space/migrate/bootstrap.ts index 7721cbd..b28eec1 100644 --- a/packages/database/space/migrate/bootstrap.ts +++ b/packages/database/space/migrate/bootstrap.ts @@ -21,10 +21,8 @@ export async function bootstrapSpaceDatabase( lockTimeout: string = "5s", transactionTimeout: string = "30s", idleInTransactionSessionTimeout: string = "30s", - shardId?: number, ): Promise { const attributes = { - "db.shard": shardId, "db.statement_timeout": statementTimeout, "db.lock_timeout": lockTimeout, "db.transaction_timeout": transactionTimeout, @@ -40,9 +38,6 @@ export async function bootstrapSpaceDatabase( try { const [key1, key2] = advisoryLockKey("memory-space:bootstrap"); await sql.begin(async (tx) => { - if (shardId !== undefined) { - await tx.unsafe(`set local pgdog.shard to ${String(shardId)}`); - } await ensurePostgresVersion(tx); const acquired = await span("space.bootstrap.acquire_lock", { callback: () => acquireAdvisoryLock(tx, key1, key2), diff --git a/packages/database/space/migrate/migrate.ts b/packages/database/space/migrate/migrate.ts index 214df0a..660e0d8 100644 --- a/packages/database/space/migrate/migrate.ts +++ b/packages/database/space/migrate/migrate.ts @@ -61,7 +61,6 @@ export interface MigrateSpaceOptions { */ schema?: string; logSqlFiles?: boolean; - shardId?: number; embeddingDimensions?: number; bm25TextConfig?: string; bm25K1?: number; @@ -79,7 +78,6 @@ interface NormalizedMigrateSpaceOptions { schema?: string; logSqlFiles: boolean; schemaVersion: string; - shardId?: number; embeddingDimensions: number; bm25TextConfig: string; bm25K1: number; @@ -121,14 +119,6 @@ export async function migrateSpace( const [key1, key2] = advisoryLockKey(`memory-space:schema:${schema}`); await sql.begin(async (tx) => { - if (opts.shardId !== undefined) { - if (!Number.isSafeInteger(opts.shardId)) { - throw new Error( - `shardId must be a safe integer, got: ${opts.shardId}`, - ); - } - await tx.unsafe(`set local pgdog.shard to ${String(opts.shardId)}`); - } await applySessionTimeouts(tx, opts); const acquired = await span("space.migrate.acquire_lock", { attributes: schemaAttributes, @@ -189,7 +179,6 @@ function migrateAttributes( return { "space.slug": options.slug, "space.schema_version": options.schemaVersion, - "db.shard": options.shardId, "db.statement_timeout": options.statementTimeout, "db.lock_timeout": options.lockTimeout, "db.transaction_timeout": options.transactionTimeout, @@ -206,7 +195,6 @@ function normalizeMigrateSpaceOptions( schema: options.schema, logSqlFiles: options.logSqlFiles ?? false, schemaVersion: SPACE_SCHEMA_VERSION, - shardId: options.shardId, embeddingDimensions: options.embeddingDimensions ?? 1536, bm25TextConfig: options.bm25TextConfig ?? "english", bm25K1: options.bm25K1 ?? 1.2, From b1e7382c3b0f8aa1cf7a19f510c3e7e6e0049528 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 10:50:36 +0200 Subject: [PATCH 028/156] refactor(accounts): drop OAuth-token storage + encryption subsystem (login-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OAuth here is authentication-only: the provider access/refresh tokens were stored (envelope-encrypted) but never read back — getOAuthTokens and refreshOAuthTokens had zero callers. Remove the dead at-rest-secret machinery: - delete util/crypto.ts (AES-256-GCM envelope encryption), the AccountsCrypto context field, createDataKey/activateDataKey, and the ACCOUNTS_MASTER_KEY requirement + startup data-key creation. - linkOAuthAccount no longer accepts or persists tokens; drop getOAuthTokens and refreshOAuthTokens. - the OAuth callback still uses the access token once to fetch the user's identity, then discards it. - drop the now-unused ACCOUNTS_MASTER_KEY from .env.sample and DEVELOPMENT.md. The oauth_account token columns and the encryption_key table remain in the legacy accounts schema and go away when that schema is retired. The generate:master-key script is left for the package.json cleanup. Verified: typecheck, lint, 1017 unit + 63 accounts/server integration tests (local PG18). Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.sample | 4 - DEVELOPMENT.md | 12 - packages/accounts/db.integration.test.ts | 96 +------- packages/accounts/db.ts | 34 +-- packages/accounts/index.ts | 6 +- packages/accounts/ops/oauth.ts | 97 +------- packages/accounts/types.ts | 25 +- packages/accounts/util/crypto.ts | 222 ------------------ packages/server/handlers/auth.ts | 8 +- packages/server/index.ts | 28 +-- .../rpc/accounts/engine.integration.test.ts | 14 +- .../rpc/accounts/org.integration.test.ts | 12 +- 12 files changed, 20 insertions(+), 538 deletions(-) delete mode 100644 packages/accounts/util/crypto.ts diff --git a/.env.sample b/.env.sample index 7ab64b6..9237129 100644 --- a/.env.sample +++ b/.env.sample @@ -11,10 +11,6 @@ # (stores identities, orgs, engines, sessions, API keys) ACCOUNTS_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me -# 32-byte hex key for encrypting API keys at rest -# Generate with: openssl rand -hex 32 -ACCOUNTS_MASTER_KEY= - # PostgreSQL connection string for the engine database # (stores memories — each engine gets its own schema) ENGINE_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ff80e6b..5a0a9f4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -59,17 +59,6 @@ ACCOUNTS_DATABASE_URL=postgres://postgres@localhost:5432/accounts ENGINE_DATABASE_URL=postgres://postgres@localhost:5432/shard1 ``` -**Encryption master key** — 32-byte hex string for encrypting API keys at rest: - -```bash -./bun run generate:master-key -``` - -Paste the output into `.env`: - -``` -ACCOUNTS_MASTER_KEY= -``` **Server URL and port** — `API_BASE_URL` is used to construct OAuth callback URLs. `PORT` controls which port the server listens on. They must be consistent: @@ -125,7 +114,6 @@ GITHUB_CLIENT_SECRET=... # Database ACCOUNTS_DATABASE_URL=postgres://postgres@localhost:5432/accounts ENGINE_DATABASE_URL=postgres://postgres@localhost:5432/shard1 -ACCOUNTS_MASTER_KEY=<./bun run generate:master-key> # Server API_BASE_URL=http://localhost:3000 diff --git a/packages/accounts/db.integration.test.ts b/packages/accounts/db.integration.test.ts index f15700a..e420800 100644 --- a/packages/accounts/db.integration.test.ts +++ b/packages/accounts/db.integration.test.ts @@ -3,25 +3,13 @@ import { type AccountsDB, createAccountsDB } from "./db"; import { TestDatabase } from "./migrate/test-utils"; import { AccountsError } from "./types"; -// Test master key (32 bytes for AES-256) -const TEST_MASTER_KEY = Buffer.from( - "0123456789abcdef0123456789abcdef", - "utf-8", -); - let testDb: TestDatabase; let db: AccountsDB; beforeAll(async () => { testDb = await TestDatabase.create(); - db = createAccountsDB(testDb.sql, testDb.schema, { - masterKey: TEST_MASTER_KEY, - }); - - // Create and activate an encryption key for tests - const keyId = await db.createDataKey(); - await db.activateDataKey(keyId); + db = createAccountsDB(testDb.sql, testDb.schema); }); afterAll(async () => { @@ -535,8 +523,6 @@ describe("oauth", () => { provider: "github", providerAccountId: "gh-123", email: "oauth@example.com", - accessToken: "access-token-123", - refreshToken: "refresh-token-456", }); expect(oauth.provider).toBe("github"); @@ -546,49 +532,6 @@ describe("oauth", () => { expect(fetched?.id).toBe(oauth.id); }); - test("get decrypted tokens", async () => { - const identity = await db.createIdentity({ - email: "oauthtokens@example.com", - name: "OAuth Tokens", - }); - - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "google", - providerAccountId: "google-abc", - accessToken: "my-access-token", - refreshToken: "my-refresh-token", - }); - - const tokens = await db.getOAuthTokens(oauth.id); - expect(tokens?.accessToken).toBe("my-access-token"); - expect(tokens?.refreshToken).toBe("my-refresh-token"); - }); - - test("refresh oauth tokens", async () => { - const identity = await db.createIdentity({ - email: "refreshtokens@example.com", - name: "Refresh Tokens", - }); - - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-refresh", - accessToken: "old-access", - refreshToken: "old-refresh", - }); - - await db.refreshOAuthTokens(oauth.id, { - accessToken: "new-access", - refreshToken: "new-refresh", - }); - - const tokens = await db.getOAuthTokens(oauth.id); - expect(tokens?.accessToken).toBe("new-access"); - expect(tokens?.refreshToken).toBe("new-refresh"); - }); - test("list oauth accounts by identity", async () => { const identity = await db.createIdentity({ email: "multioauth@example.com", @@ -599,13 +542,11 @@ describe("oauth", () => { identityId: identity.id, provider: "github", providerAccountId: "gh-multi", - accessToken: "token1", }); await db.linkOAuthAccount({ identityId: identity.id, provider: "google", providerAccountId: "google-multi", - accessToken: "token2", }); const accounts = await db.getOAuthAccountsByIdentity(identity.id); @@ -613,41 +554,6 @@ describe("oauth", () => { }); }); -// --------------------------------------------------------------------------- -// Encryption key rotation test -// --------------------------------------------------------------------------- - -describe("encryption key rotation", () => { - test("can rotate encryption keys", async () => { - const identity = await db.createIdentity({ - email: "rotation@example.com", - name: "Rotation Test", - }); - - // Link with current key - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-rotation", - accessToken: "original-token", - }); - - // Create and activate new key - const newKeyId = await db.createDataKey(); - await db.activateDataKey(newKeyId); - - // Old tokens should still decrypt (using old key stored with them) - const tokens = await db.getOAuthTokens(oauth.id); - expect(tokens?.accessToken).toBe("original-token"); - - // New tokens will use the new key - await db.refreshOAuthTokens(oauth.id, { accessToken: "rotated-token" }); - - const newTokens = await db.getOAuthTokens(oauth.id); - expect(newTokens?.accessToken).toBe("rotated-token"); - }); -}); - // --------------------------------------------------------------------------- // Transaction test // --------------------------------------------------------------------------- diff --git a/packages/accounts/db.ts b/packages/accounts/db.ts index 9fb02a2..aaf74ca 100644 --- a/packages/accounts/db.ts +++ b/packages/accounts/db.ts @@ -20,11 +20,6 @@ import { setLocalAccountsTimeouts, } from "./ops"; import type { AccountsContext } from "./types"; -import { createAccountsCrypto } from "./util/crypto"; - -export interface CreateAccountsDBOptions { - masterKey: Buffer; -} type AllOps = DeviceAuthOps & IdentityOps & @@ -36,10 +31,6 @@ type AllOps = DeviceAuthOps & SessionOps; export interface AccountsDB extends AllOps { - /** Create a new encryption data key (does not activate) */ - createDataKey(): Promise; - /** Activate an encryption data key */ - activateDataKey(keyId: number): Promise; /** Execute operations within a transaction */ withTransaction(fn: (db: AccountsDB) => Promise): Promise; } @@ -57,18 +48,11 @@ function composeOps(ctx: AccountsContext): AllOps { }; } -export function createAccountsDB( - sql: SQL, - schema: string, - options: CreateAccountsDBOptions, -): AccountsDB { - const crypto = createAccountsCrypto(options.masterKey, { sql, schema }); - +export function createAccountsDB(sql: SQL, schema: string): AccountsDB { const ctx: AccountsContext = { sql, schema, inTransaction: false, - crypto, }; const ops = composeOps(ctx); @@ -76,30 +60,16 @@ export function createAccountsDB( const db: AccountsDB = { ...ops, - createDataKey(): Promise { - return crypto.createDataKey(); - }, - - activateDataKey(keyId: number): Promise { - return crypto.activateDataKey(keyId); - }, - async withTransaction(fn: (db: AccountsDB) => Promise): Promise { return sql.begin(async (tx) => { await setLocalAccountsTimeouts(tx); await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - const txCrypto = createAccountsCrypto(options.masterKey, { - sql: tx, - schema, - }); - const txCtx = deriveContext({ ...ctx, crypto: txCrypto }, tx); + const txCtx = deriveContext(ctx, tx); const txOps = composeOps(txCtx); const txDb: AccountsDB = { ...txOps, - createDataKey: () => txCrypto.createDataKey(), - activateDataKey: (keyId) => txCrypto.activateDataKey(keyId), withTransaction: (nestedFn: (db: AccountsDB) => Promise) => nestedFn(txDb), }; diff --git a/packages/accounts/index.ts b/packages/accounts/index.ts index da9c2b1..a1b80db 100644 --- a/packages/accounts/index.ts +++ b/packages/accounts/index.ts @@ -1,8 +1,4 @@ -export { - type AccountsDB, - type CreateAccountsDBOptions, - createAccountsDB, -} from "./db"; +export { type AccountsDB, createAccountsDB } from "./db"; export * from "./types"; export { generateToken, tokenHash } from "./util/hash"; export { generateSlug } from "./util/slug"; diff --git a/packages/accounts/ops/oauth.ts b/packages/accounts/ops/oauth.ts index 34d80d7..177d2c3 100644 --- a/packages/accounts/ops/oauth.ts +++ b/packages/accounts/ops/oauth.ts @@ -12,10 +12,6 @@ interface OAuthAccountRow { provider: OAuthProvider; provider_account_id: string; email: string | null; - access_token: string | null; - refresh_token: string | null; - encryption_key_id: number | null; - token_expires_at: Date | null; created_at: Date; updated_at: Date | null; } @@ -33,42 +29,26 @@ function rowToOAuthAccount(row: OAuthAccountRow): OAuthAccount { } export function oauthOps(ctx: AccountsContext) { - const { schema, crypto } = ctx; + const { schema } = ctx; return { + // Login-only: we use the provider access token once during the OAuth + // callback to fetch the user's identity, then discard it. No provider + // tokens are persisted, so there is nothing to encrypt at rest. async linkOAuthAccount(params: LinkOAuthParams): Promise { - const { - identityId, - provider, - providerAccountId, - email, - accessToken, - refreshToken, - tokenExpiresAt, - } = params; - - // Encrypt tokens - const { ciphertext: encryptedAccess, keyId } = - await crypto.encrypt(accessToken); - const encryptedRefresh = refreshToken - ? (await crypto.encrypt(refreshToken)).ciphertext - : null; + const { identityId, provider, providerAccountId, email } = params; return withTx(ctx, "linkOAuthAccount", async (sql) => { const rows = await sql` insert into ${sql.unsafe(schema)}.oauth_account - (identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at) - values (${identityId}, ${provider}, ${providerAccountId}, ${email ?? null}, ${encryptedAccess}, ${encryptedRefresh}, ${keyId}, ${tokenExpiresAt ?? null}) + (identity_id, provider, provider_account_id, email) + values (${identityId}, ${provider}, ${providerAccountId}, ${email ?? null}) on conflict (provider, provider_account_id) do update set identity_id = excluded.identity_id, email = excluded.email, - access_token = excluded.access_token, - refresh_token = excluded.refresh_token, - encryption_key_id = excluded.encryption_key_id, - token_expires_at = excluded.token_expires_at, updated_at = now() - returning id, identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at, created_at, updated_at + returning id, identity_id, provider, provider_account_id, email, created_at, updated_at `; const row = rows[0]; if (!row) { @@ -84,7 +64,7 @@ export function oauthOps(ctx: AccountsContext) { ): Promise { return withTx(ctx, "getOAuthAccount", async (sql) => { const [row] = await sql` - select id, identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at, created_at, updated_at + select id, identity_id, provider, provider_account_id, email, created_at, updated_at from ${sql.unsafe(schema)}.oauth_account where provider = ${provider} and provider_account_id = ${providerAccountId} `; @@ -97,7 +77,7 @@ export function oauthOps(ctx: AccountsContext) { ): Promise { return withTx(ctx, "getOAuthAccountsByIdentity", async (sql) => { const rows = await sql` - select id, identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at, created_at, updated_at + select id, identity_id, provider, provider_account_id, email, created_at, updated_at from ${sql.unsafe(schema)}.oauth_account where identity_id = ${identityId} order by created_at @@ -115,63 +95,6 @@ export function oauthOps(ctx: AccountsContext) { return result.count > 0; }); }, - - async refreshOAuthTokens( - id: string, - params: { - accessToken: string; - refreshToken?: string; - tokenExpiresAt?: Date; - }, - ): Promise { - const { accessToken, refreshToken, tokenExpiresAt } = params; - - const { ciphertext: encryptedAccess, keyId } = - await crypto.encrypt(accessToken); - const encryptedRefresh = refreshToken - ? (await crypto.encrypt(refreshToken)).ciphertext - : undefined; - - return withTx(ctx, "refreshOAuthTokens", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.oauth_account - set - access_token = ${encryptedAccess}, - ${encryptedRefresh !== undefined ? sql`refresh_token = ${encryptedRefresh},` : sql``} - encryption_key_id = ${keyId}, - ${tokenExpiresAt !== undefined ? sql`token_expires_at = ${tokenExpiresAt},` : sql``} - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async getOAuthTokens( - id: string, - ): Promise<{ accessToken: string; refreshToken: string | null } | null> { - return withTx(ctx, "getOAuthTokens", async (sql) => { - const [row] = await sql` - select access_token, refresh_token, encryption_key_id - from ${sql.unsafe(schema)}.oauth_account - where id = ${id} - `; - - if (!row?.access_token || !row.encryption_key_id) { - return null; - } - - const accessToken = await crypto.decrypt( - row.access_token, - row.encryption_key_id, - ); - const refreshToken = row.refresh_token - ? await crypto.decrypt(row.refresh_token, row.encryption_key_id) - : null; - - return { accessToken, refreshToken }; - }); - }, }; } diff --git a/packages/accounts/types.ts b/packages/accounts/types.ts index e11b732..764d300 100644 --- a/packages/accounts/types.ts +++ b/packages/accounts/types.ts @@ -4,18 +4,10 @@ import type { SQL } from "bun"; // Context // ============================================================================= -export interface AccountsCrypto { - encrypt(plaintext: string): Promise<{ ciphertext: string; keyId: number }>; - decrypt(ciphertext: string, keyId: number): Promise; - createDataKey(): Promise; - activateDataKey(keyId: number): Promise; -} - export interface AccountsContext { sql: SQL; schema: string; inTransaction: boolean; - crypto: AccountsCrypto; } // ============================================================================= @@ -34,9 +26,7 @@ export type AccountsErrorCode = | "SESSION_EXPIRED" | "OAUTH_ACCOUNT_NOT_FOUND" | "DUPLICATE_SLUG" - | "DUPLICATE_EMAIL" - | "ENCRYPTION_KEY_NOT_FOUND" - | "NO_ACTIVE_ENCRYPTION_KEY"; + | "DUPLICATE_EMAIL"; export class AccountsError extends Error { constructor( @@ -173,9 +163,6 @@ export interface LinkOAuthParams { provider: OAuthProvider; providerAccountId: string; email?: string; - accessToken: string; - refreshToken?: string; - tokenExpiresAt?: Date; } // ============================================================================= @@ -199,16 +186,6 @@ export interface CreateSessionResult { rawToken: string; } -// ============================================================================= -// EncryptionKey -// ============================================================================= - -export interface EncryptionKey { - id: number; - active: boolean; - createdAt: Date; -} - // ============================================================================= // DeviceAuthorization (OAuth Device Flow) // ============================================================================= diff --git a/packages/accounts/util/crypto.ts b/packages/accounts/util/crypto.ts deleted file mode 100644 index f8a2338..0000000 --- a/packages/accounts/util/crypto.ts +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Envelope encryption for OAuth tokens - * - * Uses a master key (from environment) to encrypt data keys stored in the DB. - * Data keys encrypt the actual OAuth tokens. This allows key rotation without - * re-encrypting all tokens at once. - * - * Algorithm: AES-256-GCM - * Ciphertext format: {iv}:{ciphertext}:{authTag} (all base64) - */ - -import type { SQL } from "bun"; -import type { AccountsCrypto } from "../types"; - -const ALGORITHM = "AES-GCM"; -const KEY_LENGTH = 256; -const IV_LENGTH = 12; -const TAG_LENGTH = 128; - -interface EncryptionKeyRow { - id: number; - key_ciphertext: Buffer; - active: boolean; - created_at: Date; -} - -/** - * Import a raw key for AES-GCM operations - */ -async function importKey(keyBytes: Buffer | Uint8Array): Promise { - // Copy into a fresh ArrayBuffer for Web Crypto API compatibility - // (Buffer's underlying buffer may be SharedArrayBuffer which Web Crypto rejects) - const bytes = new Uint8Array(keyBytes.length); - bytes.set(keyBytes); - - return crypto.subtle.importKey( - "raw", - bytes, - { name: ALGORITHM, length: KEY_LENGTH }, - false, - ["encrypt", "decrypt"], - ); -} - -/** - * Encrypt plaintext using AES-256-GCM - */ -async function encryptAES(key: CryptoKey, plaintext: string): Promise { - const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH)); - const encoder = new TextEncoder(); - const data = encoder.encode(plaintext); - - const ciphertext = await crypto.subtle.encrypt( - { name: ALGORITHM, iv, tagLength: TAG_LENGTH }, - key, - data, - ); - - // Split ciphertext and auth tag (last 16 bytes) - const ciphertextBytes = new Uint8Array(ciphertext.slice(0, -16)); - const authTag = new Uint8Array(ciphertext.slice(-16)); - - const ivB64 = btoa(String.fromCharCode(...iv)); - const ciphertextB64 = btoa(String.fromCharCode(...ciphertextBytes)); - const tagB64 = btoa(String.fromCharCode(...authTag)); - - return `${ivB64}:${ciphertextB64}:${tagB64}`; -} - -/** - * Decrypt ciphertext using AES-256-GCM - */ -async function decryptAES(key: CryptoKey, ciphertext: string): Promise { - const [ivB64, ciphertextB64, tagB64] = ciphertext.split(":"); - if (!ivB64 || !ciphertextB64 || !tagB64) { - throw new Error("Invalid ciphertext format"); - } - - const iv = Uint8Array.from(atob(ivB64), (c) => c.charCodeAt(0)); - const ciphertextBytes = Uint8Array.from(atob(ciphertextB64), (c) => - c.charCodeAt(0), - ); - const authTag = Uint8Array.from(atob(tagB64), (c) => c.charCodeAt(0)); - - // Combine ciphertext and auth tag for Web Crypto API - const combined = new Uint8Array(ciphertextBytes.length + authTag.length); - combined.set(ciphertextBytes); - combined.set(authTag, ciphertextBytes.length); - - const plaintext = await crypto.subtle.decrypt( - { name: ALGORITHM, iv, tagLength: TAG_LENGTH }, - key, - combined, - ); - - return new TextDecoder().decode(plaintext); -} - -/** - * Create an AccountsCrypto instance for envelope encryption - */ -export function createAccountsCrypto( - masterKey: Buffer, - ctx: { sql: SQL; schema: string }, -): AccountsCrypto { - // Cache for decrypted data keys - const keyCache = new Map(); - let masterCryptoKey: CryptoKey | null = null; - - async function getMasterKey(): Promise { - if (!masterCryptoKey) { - masterCryptoKey = await importKey(masterKey); - } - return masterCryptoKey; - } - - async function getDataKey(keyId: number): Promise { - const cached = keyCache.get(keyId); - if (cached) { - return cached; - } - - const { sql, schema } = ctx; - const [row] = await sql` - select id, key_ciphertext, active, created_at - from ${sql.unsafe(schema)}.encryption_key - where id = ${keyId} - `; - - if (!row) { - throw new Error(`Encryption key ${keyId} not found`); - } - - const master = await getMasterKey(); - const dataKeyBytes = await decryptAES( - master, - row.key_ciphertext.toString("utf-8"), - ); - const dataKey = await importKey(Buffer.from(dataKeyBytes, "base64")); - - keyCache.set(keyId, dataKey); - return dataKey; - } - - async function getActiveKeyId(): Promise { - const { sql, schema } = ctx; - const [row] = await sql<{ id: number }[]>` - select id from ${sql.unsafe(schema)}.encryption_key - where active = true - `; - - if (!row) { - throw new Error("No active encryption key"); - } - - return row.id; - } - - return { - async encrypt( - plaintext: string, - ): Promise<{ ciphertext: string; keyId: number }> { - const keyId = await getActiveKeyId(); - const dataKey = await getDataKey(keyId); - const ciphertext = await encryptAES(dataKey, plaintext); - return { ciphertext, keyId }; - }, - - async decrypt(ciphertext: string, keyId: number): Promise { - const dataKey = await getDataKey(keyId); - return decryptAES(dataKey, ciphertext); - }, - - async createDataKey(): Promise { - const { sql, schema } = ctx; - - // Generate a random 256-bit key - const dataKeyBytes = crypto.getRandomValues(new Uint8Array(32)); - const dataKeyB64 = btoa(String.fromCharCode(...dataKeyBytes)); - - // Encrypt with master key - const master = await getMasterKey(); - const keyCiphertext = await encryptAES(master, dataKeyB64); - - const [row] = await sql<{ id: number }[]>` - insert into ${sql.unsafe(schema)}.encryption_key (key_ciphertext, active) - values (${keyCiphertext}::bytea, false) - returning id - `; - - if (!row) { - throw new Error("Failed to create encryption key"); - } - - return row.id; - }, - - async activateDataKey(keyId: number): Promise { - const { sql, schema } = ctx; - - // Deactivate all keys, then activate the specified one - await sql` - update ${sql.unsafe(schema)}.encryption_key - set active = false - where active = true - `; - - const result = await sql` - update ${sql.unsafe(schema)}.encryption_key - set active = true - where id = ${keyId} - `; - - if (result.count === 0) { - throw new Error(`Encryption key ${keyId} not found`); - } - - // Clear cache since active key changed - keyCache.clear(); - }, - }; -} diff --git a/packages/server/handlers/auth.ts b/packages/server/handlers/auth.ts index 791a2d3..7a14c74 100644 --- a/packages/server/handlers/auth.ts +++ b/packages/server/handlers/auth.ts @@ -347,17 +347,13 @@ export async function oauthCallbackHandler( await provisionPersonalAccount(ctx, identity); } - // Link OAuth account (upserts if exists) + // Link OAuth account (upserts if exists). Login-only: we do not persist + // the provider tokens — they were used above only to fetch the identity. await ctx.db.linkOAuthAccount({ identityId: identity.id, provider, providerAccountId: userInfo.providerAccountId, email: userInfo.email, - accessToken: tokens.accessToken, - refreshToken: tokens.refreshToken ?? undefined, - tokenExpiresAt: tokens.expiresIn - ? new Date(Date.now() + tokens.expiresIn * 1000) - : undefined, }); // Mark device as authorized diff --git a/packages/server/index.ts b/packages/server/index.ts index c3f07d7..ead57ea 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -63,8 +63,6 @@ configure({ // Required: // ACCOUNTS_DATABASE_URL - PostgreSQL connection string for accounts database // (stores engines, API keys, users) -// ACCOUNTS_MASTER_KEY - 32-byte hex string for encrypting API keys at rest -// Generate with: openssl rand -hex 32 // ENGINE_DATABASE_URL - PostgreSQL connection string for engine databases // (stores memories, each engine in its own schema) // API_BASE_URL - Public URL for OAuth callbacks @@ -144,11 +142,6 @@ if (!accountsDatabaseUrl) { throw new Error("ACCOUNTS_DATABASE_URL environment variable is required"); } -const accountsMasterKey = process.env.ACCOUNTS_MASTER_KEY; -if (!accountsMasterKey) { - throw new Error("ACCOUNTS_MASTER_KEY environment variable is required"); -} - const engineDatabaseUrl = process.env.ENGINE_DATABASE_URL; if (!engineDatabaseUrl) { throw new Error("ENGINE_DATABASE_URL environment variable is required"); @@ -337,14 +330,6 @@ if (configuredProviders.length === 0) { // Database Pools // ============================================================================= -// Parse master key from hex string to Buffer -const masterKeyBuffer = Buffer.from(accountsMasterKey, "hex"); -if (masterKeyBuffer.length !== 32) { - throw new Error( - "ACCOUNTS_MASTER_KEY must be a 32-byte (64 character) hex string", - ); -} - // Create database connection pools const accountsSql = new Bun.SQL(accountsDatabaseUrl, { max: accountsPoolMax, @@ -368,9 +353,7 @@ const workerEngineSql = new Bun.SQL(workerEngineDatabaseUrl, { }); // Create accounts DB with operations layer -const accountsDb = createAccountsDB(accountsSql, accountsSchema, { - masterKey: masterKeyBuffer, -}); +const accountsDb = createAccountsDB(accountsSql, accountsSchema); // ============================================================================= // Database Bootstrap & Migrations (blocking — server won't serve until current) @@ -401,15 +384,6 @@ if (accountsMigrateResult.applied.length > 0) { info("Accounts schema up to date"); } -// Ensure encryption data key exists (idempotent) -try { - const keyId = await accountsDb.createDataKey(); - await accountsDb.activateDataKey(keyId); - info("Encryption data key created", { keyId }); -} catch { - // Key already exists — expected on subsequent startups -} - // Migrate all engine schemas const engineSchemas = await discoverEngineSchemas(engineSql); if (engineSchemas.length > 0) { diff --git a/packages/server/rpc/accounts/engine.integration.test.ts b/packages/server/rpc/accounts/engine.integration.test.ts index ece1bb1..bbbfc05 100644 --- a/packages/server/rpc/accounts/engine.integration.test.ts +++ b/packages/server/rpc/accounts/engine.integration.test.ts @@ -23,12 +23,6 @@ import type { HandlerContext } from "../types"; import { engineMethods } from "./engine"; import type { AccountsRpcContext } from "./types"; -// Test master key (32 bytes for AES-256) -const TEST_MASTER_KEY = Buffer.from( - "0123456789abcdef0123456789abcdef", - "utf-8", -); - // Test fixtures let accountsTestDb: AccountsTestDatabase; let engineTestDb: EngineTestDatabase; @@ -43,13 +37,7 @@ let testIdentity: Identity; beforeAll(async () => { // Set up accounts database accountsTestDb = await AccountsTestDatabase.create(); - accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema, { - masterKey: TEST_MASTER_KEY, - }); - - // Create and activate encryption key - const keyId = await accountsDb.createDataKey(); - await accountsDb.activateDataKey(keyId); + accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema); // Set up engine database engineTestDb = new EngineTestDatabase(); diff --git a/packages/server/rpc/accounts/org.integration.test.ts b/packages/server/rpc/accounts/org.integration.test.ts index dd7b71f..c11a6ce 100644 --- a/packages/server/rpc/accounts/org.integration.test.ts +++ b/packages/server/rpc/accounts/org.integration.test.ts @@ -17,23 +17,13 @@ import type { HandlerContext } from "../types"; import { orgMethods } from "./org"; import type { AccountsRpcContext } from "./types"; -const TEST_MASTER_KEY = Buffer.from( - "0123456789abcdef0123456789abcdef", - "utf-8", -); - let accountsTestDb: AccountsTestDatabase; let accountsDb: AccountsDB; let testIdentity: Identity; beforeAll(async () => { accountsTestDb = await AccountsTestDatabase.create(); - accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema, { - masterKey: TEST_MASTER_KEY, - }); - - const keyId = await accountsDb.createDataKey(); - await accountsDb.activateDataKey(keyId); + accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema); testIdentity = await accountsDb.createIdentity({ email: "org-rpc-test@example.com", From 7c126b9b458e320a2a48b1945775ce30186f2cef Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 14:32:56 +0200 Subject: [PATCH 029/156] feat(database): add core control-plane SQL functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the core schema's mutation + bridge functions, mirroring the space data-plane convention (logic lives in SQL functions; security invoker; search_path set). The TS layer (next) will only call these, never query tables directly. - build_tree_access(_member_id, _space_id) -> jsonb: the bridge from core's access model to the space functions — resolves a user/agent's effective grants and returns the [{tree_path, access}] shape that space.search_memory consumes. - create_space / get_space (no status — all spaces are active). - create_user (caller-supplied id == auth.users.id) / create_agent / create_group / get_principal. - add_principal_to_space / remove_principal_from_space (cascades the principal's grants + group memberships in the space) / add_group_member / remove_group_member. - grant_tree_access / remove_tree_access_grant (additive grants, no deny). - create_api_key (stores caller-hashed secret) / validate_api_key (hash compare + expiry, in SQL). Upserts let the before-update trigger maintain updated_at. Integration tests cover the user + group access paths, cascade-on-remove, and api-key validation (good/wrong-secret/expired). 24 tests green on local PG18. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../migrate/idempotent/003_tree_access.sql | 41 +++ .../core/migrate/idempotent/004_space.sql | 38 +++ .../core/migrate/idempotent/005_principal.sql | 76 +++++ .../migrate/idempotent/006_membership.sql | 100 ++++++ .../core/migrate/idempotent/007_grant.sql | 46 +++ .../core/migrate/idempotent/008_api_key.sql | 43 +++ .../core/migrate/migrate.integration.test.ts | 318 ++++++++++++++++++ packages/database/core/migrate/migrate.ts | 26 ++ 8 files changed, 688 insertions(+) create mode 100644 packages/database/core/migrate/idempotent/004_space.sql create mode 100644 packages/database/core/migrate/idempotent/005_principal.sql create mode 100644 packages/database/core/migrate/idempotent/006_membership.sql create mode 100644 packages/database/core/migrate/idempotent/007_grant.sql create mode 100644 packages/database/core/migrate/idempotent/008_api_key.sql diff --git a/packages/database/core/migrate/idempotent/003_tree_access.sql b/packages/database/core/migrate/idempotent/003_tree_access.sql index 7dfe20f..bc55608 100644 --- a/packages/database/core/migrate/idempotent/003_tree_access.sql +++ b/packages/database/core/migrate/idempotent/003_tree_access.sql @@ -126,3 +126,44 @@ as $func$ group by x.tree_path $func$ language sql stable security invoker ; + +------------------------------------------------------------------------------- +-- build_tree_access +-- +-- The bridge from core's access model to the space data-plane functions: +-- resolves a member's (user or agent) effective grants in a space and returns +-- them as the jsonb array shape that space.search_memory / *_memory consume via +-- jsonb_to_recordset(...) x(tree_path ltree, access int). +------------------------------------------------------------------------------- +create or replace function {{schema}}.build_tree_access +( _member_id uuid +, _space_id uuid +) +returns jsonb +as $func$ + with access as + ( + select ta.tree_path, ta.access + from {{schema}}.principal p + cross join lateral + ( + -- dispatch on kind; the off-kind branch's id column is null -> no rows + select uta.tree_path, uta.access + from {{schema}}.user_tree_access(p.user_id, _space_id) uta + where p.kind = 'u' + union all + select ata.tree_path, ata.access + from {{schema}}.agent_tree_access(p.agent_id, _space_id) ata + where p.kind = 'a' + ) ta + where p.member_id = _member_id + ) + select coalesce + ( + jsonb_agg(jsonb_build_object('tree_path', a.tree_path::text, 'access', a.access)) + , '[]'::jsonb + ) + from access a +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/004_space.sql b/packages/database/core/migrate/idempotent/004_space.sql new file mode 100644 index 0000000..33548f9 --- /dev/null +++ b/packages/database/core/migrate/idempotent/004_space.sql @@ -0,0 +1,38 @@ +------------------------------------------------------------------------------- +-- create_space +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_space +( _slug text +, _name text +, _language text default 'english' +) +returns uuid +as $func$ + insert into {{schema}}.space (slug, name, language) + values (_slug, _name, coalesce(_language, 'english')) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_space +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_space +( _slug text +) +returns table +( id uuid +, slug text +, name text +, language text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select s.id, s.slug, s.name::text, s.language, s.created_at, s.updated_at + from {{schema}}.space s + where s.slug = _slug +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/005_principal.sql b/packages/database/core/migrate/idempotent/005_principal.sql new file mode 100644 index 0000000..5d41932 --- /dev/null +++ b/packages/database/core/migrate/idempotent/005_principal.sql @@ -0,0 +1,76 @@ +------------------------------------------------------------------------------- +-- create_user +-- Users are global (no space_id, no owner_id). The id is supplied by the caller +-- so it equals auth.users.id (one identity across auth + core). +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_user +( _id uuid +, _name text +) +returns uuid +as $func$ + insert into {{schema}}.principal (id, kind, name) + values (_id, 'u', _name) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- create_agent +-- Agents are owned by a user (owner_id -> a user principal's id) and are global. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_agent +( _owner_id uuid +, _name text +, _id uuid default null +) +returns uuid +as $func$ + insert into {{schema}}.principal (id, kind, name, owner_id) + values (coalesce(_id, uuidv7()), 'a', _name, _owner_id) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- create_group +-- Groups belong to a single space. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_group +( _space_id uuid +, _name text +, _id uuid default null +) +returns uuid +as $func$ + insert into {{schema}}.principal (id, kind, name, space_id) + values (coalesce(_id, uuidv7()), 'g', _name, _space_id) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_principal +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_principal +( _id uuid +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, space_id uuid +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.kind, p.name::text, p.owner_id, p.space_id, p.created_at, p.updated_at + from {{schema}}.principal p + where p.id = _id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/006_membership.sql b/packages/database/core/migrate/idempotent/006_membership.sql new file mode 100644 index 0000000..00986e3 --- /dev/null +++ b/packages/database/core/migrate/idempotent/006_membership.sql @@ -0,0 +1,100 @@ +------------------------------------------------------------------------------- +-- add_principal_to_space +-- Adds (or updates the admin flag of) a principal's membership in a space. +------------------------------------------------------------------------------- +create or replace function {{schema}}.add_principal_to_space +( _space_id uuid +, _principal_id uuid +, _admin bool default false +) +returns void +as $func$ + insert into {{schema}}.principal_space (space_id, principal_id, admin) + values (_space_id, _principal_id, _admin) + on conflict (principal_id, space_id) do update set + admin = excluded.admin -- updated_at maintained by the before-update trigger +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- add_group_member +-- Adds a user/agent member to a group within a space. +------------------------------------------------------------------------------- +create or replace function {{schema}}.add_group_member +( _space_id uuid +, _group_id uuid +, _member_id uuid +, _admin bool default false +) +returns void +as $func$ + insert into {{schema}}.group_member (space_id, group_id, member_id, admin) + values (_space_id, _group_id, _member_id, _admin) + on conflict (space_id, member_id, group_id) do update set + admin = excluded.admin -- updated_at maintained by the before-update trigger +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- remove_principal_from_space +-- Removes a principal from a space and cascades: scrubs its tree_access grants +-- and its group_member rows in that space (both as a member and, if it is a +-- group, its members). Returns true if the principal was a member of the space. +-- (Space-scoped only; the principal row itself and any other spaces are left +-- untouched.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.remove_principal_from_space +( _space_id uuid +, _principal_id uuid +) +returns bool +as $func$ + with del_grants as + ( + delete from {{schema}}.tree_access + where space_id = _space_id + and principal_id = _principal_id + ) + , del_group_member as + ( + delete from {{schema}}.group_member + where space_id = _space_id + and (member_id = _principal_id or group_id = _principal_id) + ) + , del_membership as + ( + delete from {{schema}}.principal_space + where space_id = _space_id + and principal_id = _principal_id + returning 1 + ) + select exists (select 1 from del_membership) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- remove_group_member +-- Removes a member from a group within a space. Returns true if a row was removed. +------------------------------------------------------------------------------- +create or replace function {{schema}}.remove_group_member +( _space_id uuid +, _group_id uuid +, _member_id uuid +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.group_member + where space_id = _space_id + and group_id = _group_id + and member_id = _member_id + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/007_grant.sql b/packages/database/core/migrate/idempotent/007_grant.sql new file mode 100644 index 0000000..f9194e1 --- /dev/null +++ b/packages/database/core/migrate/idempotent/007_grant.sql @@ -0,0 +1,46 @@ +------------------------------------------------------------------------------- +-- grant_tree_access +-- Grants (or updates) a principal's access at a tree path in a space. +-- access: 1 = read, 2 = write, 3 = owner. Access is purely additive (grants); +-- there are no deny entries. +------------------------------------------------------------------------------- +create or replace function {{schema}}.grant_tree_access +( _space_id uuid +, _principal_id uuid +, _tree_path ltree +, _access int +) +returns void +as $func$ + insert into {{schema}}.tree_access (space_id, principal_id, tree_path, access) + values (_space_id, _principal_id, _tree_path, _access) + on conflict (space_id, principal_id, tree_path) do update set + access = excluded.access -- updated_at maintained by the before-update trigger +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- remove_tree_access_grant +-- Removes a single grant. Returns true if a row was removed. (No deny perms, +-- so removing a grant simply drops that access.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.remove_tree_access_grant +( _space_id uuid +, _principal_id uuid +, _tree_path ltree +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.tree_access + where space_id = _space_id + and principal_id = _principal_id + and tree_path = _tree_path + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/008_api_key.sql b/packages/database/core/migrate/idempotent/008_api_key.sql new file mode 100644 index 0000000..3fb7990 --- /dev/null +++ b/packages/database/core/migrate/idempotent/008_api_key.sql @@ -0,0 +1,43 @@ +------------------------------------------------------------------------------- +-- create_api_key +-- The caller generates the key (lookup_id + secret) and passes the *hashed* +-- secret; we never store the plaintext. Scoped to a member (user or agent). +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_api_key +( _member_id uuid +, _lookup_id text +, _secret text -- already hashed by the caller +, _name text +, _expires_at timestamptz default null +) +returns uuid +as $func$ + insert into {{schema}}.api_key (member_id, lookup_id, secret, name, expires_at) + values (_member_id, _lookup_id, _secret, _name, _expires_at) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- validate_api_key +-- Looks a key up by lookup_id, compares the hashed secret, and enforces expiry. +-- Returns the member_id + api_key id when valid; no rows otherwise. +------------------------------------------------------------------------------- +create or replace function {{schema}}.validate_api_key +( _lookup_id text +, _secret text -- hashed +) +returns table +( member_id uuid +, api_key_id uuid +) +as $func$ + select k.member_id, k.id + from {{schema}}.api_key k + where k.lookup_id = _lookup_id + and k.secret = _secret + and (k.expires_at is null or k.expires_at > now()) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/migrate.integration.test.ts b/packages/database/core/migrate/migrate.integration.test.ts index 7b69775..2c5c423 100644 --- a/packages/database/core/migrate/migrate.integration.test.ts +++ b/packages/database/core/migrate/migrate.integration.test.ts @@ -278,6 +278,324 @@ describe("agent_tree_access clamps agent access to its owner", () => { }); }); +describe("control-plane functions", () => { + /** A fresh uuidv7 from the database (principal.id requires version 7). */ + async function v7(): Promise { + const [row] = await sql.unsafe(`select uuidv7() as id`); + return row?.id as string; + } + + type Grant = { tree_path: string; access: number }; + + test("create_space + create_user + grant → build_tree_access returns the search_memory jsonb shape", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Test Space", + ]); + const spaceId = sp?.id as string; + + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + userId, + "alice", + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + true, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + userId, + "work.projects", + 2, + ]); + + const [row] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + const ta = row?.ta as Grant[]; + expect(ta).toEqual([{ tree_path: "work.projects", access: 2 }]); + }); + }); + + test("build_tree_access includes access granted via a group", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Team Space", + ]); + const spaceId = sp?.id as string; + + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + userId, + "bob", + ]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + "engineering", + ]); + const groupId = grp?.id as string; + + // both the user and the group must be members of the space + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + groupId, + false, + ]); + await sql.unsafe(`select ${s}.add_group_member($1, $2, $3, $4)`, [ + spaceId, + groupId, + userId, + false, + ]); + // grant to the GROUP, not the user + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + groupId, + "shared.docs", + 1, + ]); + + const [row] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + const ta = row?.ta as Grant[]; + expect(ta).toContainEqual({ tree_path: "shared.docs", access: 1 }); + }); + }); + + test("remove_group_member revokes group-inherited access", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Team", + ]); + const spaceId = sp?.id as string; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "erin"]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + "ops", + ]); + const groupId = grp?.id as string; + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + groupId, + false, + ]); + await sql.unsafe(`select ${s}.add_group_member($1, $2, $3, $4)`, [ + spaceId, + groupId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + groupId, + "team.notes", + 2, + ]); + + // sanity: access is inherited via the group + const [before] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + expect(before?.ta as Grant[]).toContainEqual({ + tree_path: "team.notes", + access: 2, + }); + + const [removed] = await sql.unsafe( + `select ${s}.remove_group_member($1, $2, $3) as removed`, + [spaceId, groupId, userId], + ); + expect(removed?.removed).toBe(true); + + const [after] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + expect(after?.ta).toEqual([]); + + // second remove is a no-op + const [again] = await sql.unsafe( + `select ${s}.remove_group_member($1, $2, $3) as removed`, + [spaceId, groupId, userId], + ); + expect(again?.removed).toBe(false); + }); + }); + + test("remove_principal_from_space cascades grants + group memberships (space-scoped)", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Cascade", + ]); + const spaceId = sp?.id as string; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "frank"]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + "team", + ]); + const groupId = grp?.id as string; + + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + groupId, + false, + ]); + await sql.unsafe(`select ${s}.add_group_member($1, $2, $3, $4)`, [ + spaceId, + groupId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + userId, + "direct", + 2, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + groupId, + "shared", + 1, + ]); + + const [removed] = await sql.unsafe( + `select ${s}.remove_principal_from_space($1, $2) as removed`, + [spaceId, userId], + ); + expect(removed?.removed).toBe(true); + + const count = async (table: string, col: string, id: string) => { + const [r] = await sql.unsafe( + `select count(*)::int as n from ${s}.${table} where space_id=$1 and ${col}=$2`, + [spaceId, id], + ); + return Number(r?.n); + }; + // the user's membership, direct grant, and group membership are all gone + expect(await count("principal_space", "principal_id", userId)).toBe(0); + expect(await count("tree_access", "principal_id", userId)).toBe(0); + expect(await count("group_member", "member_id", userId)).toBe(0); + // the group itself and its own grant are untouched + expect(await count("principal_space", "principal_id", groupId)).toBe(1); + expect(await count("tree_access", "principal_id", groupId)).toBe(1); + }); + }); + + test("remove_tree_access_grant drops the grant", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Sp", + ]); + const spaceId = sp?.id as string; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "carol"]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + userId, + "a.b", + 3, + ]); + + const [first] = await sql.unsafe( + `select ${s}.remove_tree_access_grant($1, $2, $3::ltree) as removed`, + [spaceId, userId, "a.b"], + ); + expect(first?.removed).toBe(true); + const [second] = await sql.unsafe( + `select ${s}.remove_tree_access_grant($1, $2, $3::ltree) as removed`, + [spaceId, userId, "a.b"], + ); + expect(second?.removed).toBe(false); + + const [row] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + expect(row?.ta).toEqual([]); + }); + }); + + test("create_api_key + validate_api_key (good, wrong-secret, expired)", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "dave"]); + + const lookup = "abcdEFGH12345678"; // 16 chars, matches lookup_id check + await sql.unsafe(`select ${s}.create_api_key($1, $2, $3, $4)`, [ + userId, + lookup, + "hashed-secret", + "default", + ]); + + const valid = await sql.unsafe( + `select member_id from ${s}.validate_api_key($1, $2)`, + [lookup, "hashed-secret"], + ); + expect(valid.length).toBe(1); + expect(valid[0]?.member_id).toBe(userId); + + const wrong = await sql.unsafe( + `select member_id from ${s}.validate_api_key($1, $2)`, + [lookup, "nope"], + ); + expect(wrong.length).toBe(0); + + // expired key + const lookup2 = "ZYXW9876_-abcdef"; + await sql.unsafe( + `select ${s}.create_api_key($1, $2, $3, $4, $5::timestamptz)`, + [userId, lookup2, "h2", "expired", "2000-01-01T00:00:00Z"], + ); + const expired = await sql.unsafe( + `select member_id from ${s}.validate_api_key($1, $2)`, + [lookup2, "h2"], + ); + expect(expired.length).toBe(0); + }); + }); +}); + describe("migration behavior", () => { test("is idempotent: re-running changes no migration rows or version", async () => { await withTestCore(sql, {}, async (core) => { diff --git a/packages/database/core/migrate/migrate.ts b/packages/database/core/migrate/migrate.ts index b09fe0e..70a2db9 100644 --- a/packages/database/core/migrate/migrate.ts +++ b/packages/database/core/migrate/migrate.ts @@ -26,6 +26,15 @@ import idempotent002 from "./idempotent/002_group_member.sql" with { import idempotent003 from "./idempotent/003_tree_access.sql" with { type: "text", }; +import idempotent004 from "./idempotent/004_space.sql" with { type: "text" }; +import idempotent005 from "./idempotent/005_principal.sql" with { + type: "text", +}; +import idempotent006 from "./idempotent/006_membership.sql" with { + type: "text", +}; +import idempotent007 from "./idempotent/007_grant.sql" with { type: "text" }; +import idempotent008 from "./idempotent/008_api_key.sql" with { type: "text" }; import incremental001 from "./incremental/001_space.sql" with { type: "text" }; import incremental002 from "./incremental/002_principal.sql" with { type: "text", @@ -92,6 +101,23 @@ const idempotents: Migration[] = [ file: "idempotent/003_tree_access.sql", sql: idempotent003, }, + { name: "004_space", file: "idempotent/004_space.sql", sql: idempotent004 }, + { + name: "005_principal", + file: "idempotent/005_principal.sql", + sql: idempotent005, + }, + { + name: "006_membership", + file: "idempotent/006_membership.sql", + sql: idempotent006, + }, + { name: "007_grant", file: "idempotent/007_grant.sql", sql: idempotent007 }, + { + name: "008_api_key", + file: "idempotent/008_api_key.sql", + sql: idempotent008, + }, ]; /** From 913b99ca0648122bb02ac5df9cb2c129ded99563 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 14:40:55 +0200 Subject: [PATCH 030/156] feat(engine): add core control-plane TS layer (createCoreDB) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thin TS layer over the core SQL functions, in a new packages/engine/core module (exported as the `core` namespace). Every method calls a core function — no table queries in TS. - createCoreDB(sql, schema): space create/get, principal create/get (user/agent/group), space membership add/remove, group member add/remove, tree-access grant/remove, buildTreeAccess (the jsonb the space data-plane functions consume), api-key create/validate, and withTransaction. Uses postgres.js. - core/api-key.ts: key format kept (me.{spaceSlug}.{lookupId}.{secret}), but secrets are now hashed with sha256 instead of argon2. The secret is high-entropy, so a fast hash is sufficient (matching session tokens) and it lets validate_api_key compare by equality in SQL — no per-request argon2 verify. - Namespaced as `core` to avoid clashing with the legacy formatApiKey/parseApiKey during the migration. - Adds @memory.build/database + postgres to engine deps. Not wired into the server yet (that's the Phase 4 switch). 7 integration tests green on local PG18; full check (1030 unit) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- bun.lock | 2 + packages/engine/core/api-key.ts | 68 ++++++ packages/engine/core/db.integration.test.ts | 143 +++++++++++++ packages/engine/core/db.ts | 224 ++++++++++++++++++++ packages/engine/core/index.ts | 17 ++ packages/engine/core/types.ts | 55 +++++ packages/engine/index.ts | 5 + packages/engine/package.json | 4 +- 8 files changed, 517 insertions(+), 1 deletion(-) create mode 100644 packages/engine/core/api-key.ts create mode 100644 packages/engine/core/db.integration.test.ts create mode 100644 packages/engine/core/db.ts create mode 100644 packages/engine/core/index.ts create mode 100644 packages/engine/core/types.ts diff --git a/bun.lock b/bun.lock index 728b6e7..73094dc 100644 --- a/bun.lock +++ b/bun.lock @@ -102,7 +102,9 @@ "name": "@memory.build/engine", "version": "0.2.5", "dependencies": { + "@memory.build/database": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", }, }, "packages/protocol": { diff --git a/packages/engine/core/api-key.ts b/packages/engine/core/api-key.ts new file mode 100644 index 0000000..7b76eaf --- /dev/null +++ b/packages/engine/core/api-key.ts @@ -0,0 +1,68 @@ +/** + * API key helpers for the core control plane. + * + * Key format (unchanged from the legacy engine): me.{spaceSlug}.{lookupId}.{secret} + * - me fixed prefix + * - spaceSlug 12-char lowercase alphanumeric (the core.space slug — routing) + * - lookupId 16-char id for the indexed db lookup + * - secret 32-char base64url random secret + * + * The secret is high-entropy, so we store sha256(secret) and validate by + * equality in SQL (core.validate_api_key) — no per-request argon2 verify. This + * matches how session tokens are handled (see packages/accounts/util/hash.ts). + */ + +const LOOKUP_ID_LENGTH = 16; +const SECRET_LENGTH = 32; +const LOOKUP_ID_CHARSET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-"; + +/** Generate a random 16-char lookup id (matches the lookup_id check). */ +export function generateLookupId(): string { + const bytes = crypto.getRandomValues(new Uint8Array(LOOKUP_ID_LENGTH)); + let result = ""; + for (const byte of bytes) { + result += LOOKUP_ID_CHARSET[byte % LOOKUP_ID_CHARSET.length]; + } + return result; +} + +/** Generate a random 32-char base64url secret. */ +export function generateSecret(): string { + const bytes = crypto.getRandomValues(new Uint8Array(SECRET_LENGTH)); + return btoa(String.fromCharCode(...bytes)) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=/g, "") + .slice(0, SECRET_LENGTH); +} + +/** Hash a secret for storage / comparison: sha256, hex-encoded. */ +export function hashApiKeySecret(secret: string): string { + return new Bun.CryptoHasher("sha256").update(secret).digest("hex"); +} + +/** Assemble a full API key string from its parts. */ +export function formatApiKey( + spaceSlug: string, + lookupId: string, + secret: string, +): string { + return `me.${spaceSlug}.${lookupId}.${secret}`; +} + +/** Parse an API key into its components; null if malformed. */ +export function parseApiKey( + key: string, +): { spaceSlug: string; lookupId: string; secret: string } | null { + const parts = key.split("."); + if (parts.length !== 4) { + return null; + } + const [prefix, spaceSlug, lookupId, secret] = parts; + if (prefix !== "me") return null; + if (!spaceSlug || !/^[a-z0-9]{12}$/.test(spaceSlug)) return null; + if (!lookupId || !/^[A-Za-z0-9_-]{16}$/.test(lookupId)) return null; + if (!secret || secret.length !== SECRET_LENGTH) return null; + return { spaceSlug, lookupId, secret }; +} diff --git a/packages/engine/core/db.integration.test.ts b/packages/engine/core/db.integration.test.ts new file mode 100644 index 0000000..ddbfe3f --- /dev/null +++ b/packages/engine/core/db.integration.test.ts @@ -0,0 +1,143 @@ +// Integration tests for the core control-plane TS layer (createCoreDB). +// +// Provisions a throwaway `core_test_` schema via migrateCore and exercises +// the thin wrappers against the real SQL functions. Run with a database, e.g.: +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/engine/core/db.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateCore } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { formatApiKey, parseApiKey } from "./api-key"; +import { type CoreDB, createCoreDB } from "./db"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +function randomFrom(n: number): string { + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += ALPHABET[b % 36]; + return s; +} +const randomCoreSchema = () => `core_test_${randomFrom(8)}`; +const randomSlug = () => randomFrom(12); + +let sql: Sql; +let schema: string; +let db: CoreDB; + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + schema = randomCoreSchema(); + await migrateCore(sql, { schema }); + db = createCoreDB(sql, schema); +}); + +afterAll(async () => { + if (schema) await sql.unsafe(`drop schema if exists ${schema} cascade`); + await sql.end(); +}); + +/** A fresh uuidv7 (principal.id requires version 7, = the future auth.users.id). */ +async function newUserId(): Promise { + const [row] = await sql`select uuidv7() as id`; + return row?.id as string; +} + +test("createSpace + getSpace round-trips", async () => { + const slug = randomSlug(); + const id = await db.createSpace(slug, "My Space"); + expect(id).toBeTruthy(); + + const space = await db.getSpace(slug); + expect(space?.id).toBe(id); + expect(space?.name).toBe("My Space"); + expect(space?.language).toBe("english"); + + expect(await db.getSpace(randomSlug())).toBeNull(); +}); + +test("createUser + getPrincipal", async () => { + const userId = await newUserId(); + await db.createUser(userId, `alice_${userId.slice(0, 8)}`); + + const p = await db.getPrincipal(userId); + expect(p?.id).toBe(userId); + expect(p?.kind).toBe("u"); + expect(p?.ownerId).toBeNull(); + expect(p?.spaceId).toBeNull(); +}); + +test("grant + buildTreeAccess returns the search_memory jsonb shape", async () => { + const spaceId = await db.createSpace(randomSlug(), "S"); + const userId = await newUserId(); + await db.createUser(userId, `bob_${userId.slice(0, 8)}`); + await db.addPrincipalToSpace(spaceId, userId, true); + await db.grantTreeAccess(spaceId, userId, "work.projects", 2); + + const ta = await db.buildTreeAccess(userId, spaceId); + expect(ta).toEqual([{ tree_path: "work.projects", access: 2 }]); +}); + +test("group access flows through buildTreeAccess; removeGroupMember revokes it", async () => { + const spaceId = await db.createSpace(randomSlug(), "T"); + const userId = await newUserId(); + await db.createUser(userId, `carol_${userId.slice(0, 8)}`); + const groupId = await db.createGroup(spaceId, "eng"); + + await db.addPrincipalToSpace(spaceId, userId); + await db.addPrincipalToSpace(spaceId, groupId); + await db.addGroupMember(spaceId, groupId, userId); + await db.grantTreeAccess(spaceId, groupId, "shared", 1); + + expect(await db.buildTreeAccess(userId, spaceId)).toContainEqual({ + tree_path: "shared", + access: 1, + }); + + expect(await db.removeGroupMember(spaceId, groupId, userId)).toBe(true); + expect(await db.buildTreeAccess(userId, spaceId)).toEqual([]); +}); + +test("createApiKey + validateApiKey (good / wrong secret)", async () => { + const userId = await newUserId(); + await db.createUser(userId, `dave_${userId.slice(0, 8)}`); + + const key = await db.createApiKey(userId, "default"); + expect(key.lookupId).toMatch(/^[A-Za-z0-9_-]{16}$/); + expect(key.secret.length).toBe(32); + + const valid = await db.validateApiKey(key.lookupId, key.secret); + expect(valid?.memberId).toBe(userId); + expect(valid?.apiKeyId).toBe(key.id); + + expect(await db.validateApiKey(key.lookupId, "wrong-secret")).toBeNull(); +}); + +test("api key string format round-trips with parseApiKey", async () => { + const slug = randomSlug(); + const userId = await newUserId(); + await db.createUser(userId, `erin_${userId.slice(0, 8)}`); + const key = await db.createApiKey(userId, "fmt"); + + const str = formatApiKey(slug, key.lookupId, key.secret); + expect(parseApiKey(str)).toEqual({ + spaceSlug: slug, + lookupId: key.lookupId, + secret: key.secret, + }); +}); + +test("withTransaction rolls back on error", async () => { + const slug = randomSlug(); + await expect( + db.withTransaction(async (tx) => { + await tx.createSpace(slug, "Tx Space"); + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + // rolled back — the space was never committed + expect(await db.getSpace(slug)).toBeNull(); +}); diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts new file mode 100644 index 0000000..0217e35 --- /dev/null +++ b/packages/engine/core/db.ts @@ -0,0 +1,224 @@ +import { CORE_SCHEMA } from "@memory.build/database"; +import type { Sql } from "postgres"; +import { generateLookupId, generateSecret, hashApiKeySecret } from "./api-key"; +import type { + AccessLevel, + CreatedApiKey, + Principal, + PrincipalKind, + Space, + TreeAccess, + ValidatedApiKey, +} from "./types"; + +/** + * The core control-plane data layer. + * + * Thin wrappers over the core SQL functions — every method calls a function in + * packages/database/core/migrate/idempotent/*.sql; none query core tables + * directly. Access enforcement and multi-table logic live in the SQL. + */ +export interface CoreDB { + createSpace(slug: string, name: string, language?: string): Promise; + getSpace(slug: string): Promise; + + createUser(id: string, name: string): Promise; + createAgent(ownerId: string, name: string, id?: string): Promise; + createGroup(spaceId: string, name: string, id?: string): Promise; + getPrincipal(id: string): Promise; + + addPrincipalToSpace( + spaceId: string, + principalId: string, + admin?: boolean, + ): Promise; + removePrincipalFromSpace( + spaceId: string, + principalId: string, + ): Promise; + addGroupMember( + spaceId: string, + groupId: string, + memberId: string, + admin?: boolean, + ): Promise; + removeGroupMember( + spaceId: string, + groupId: string, + memberId: string, + ): Promise; + + grantTreeAccess( + spaceId: string, + principalId: string, + treePath: string, + access: AccessLevel, + ): Promise; + removeTreeAccessGrant( + spaceId: string, + principalId: string, + treePath: string, + ): Promise; + + /** Resolve a member's effective grants in a space (for the space functions). */ + buildTreeAccess(memberId: string, spaceId: string): Promise; + + /** Mint an api key for a member; returns the one-time plaintext secret. */ + createApiKey( + memberId: string, + name: string, + opts?: { expiresAt?: Date }, + ): Promise; + validateApiKey( + lookupId: string, + secret: string, + ): Promise; + + /** Run operations atomically against the same transaction. */ + withTransaction(fn: (db: CoreDB) => Promise): Promise; +} + +function mapSpace(row: Record): Space { + return { + id: row.id as string, + slug: row.slug as string, + name: row.name as string, + language: row.language as string, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapPrincipal(row: Record): Principal { + return { + id: row.id as string, + kind: row.kind as PrincipalKind, + name: row.name as string, + ownerId: (row.owner_id as string | null) ?? null, + spaceId: (row.space_id as string | null) ?? null, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +export function createCoreDB(sql: Sql, schema: string = CORE_SCHEMA): CoreDB { + const sch = sql(schema); // escaped schema identifier reused across queries + + const db: CoreDB = { + async createSpace(slug, name, language) { + const [row] = await sql` + select ${sch}.create_space(${slug}, ${name}, ${language ?? null}) as id + `; + if (!row) throw new Error("create_space returned no row"); + return row.id as string; + }, + + async getSpace(slug) { + const [row] = await sql`select * from ${sch}.get_space(${slug})`; + return row ? mapSpace(row) : null; + }, + + async createUser(id, name) { + const [row] = await sql`select ${sch}.create_user(${id}, ${name}) as id`; + if (!row) throw new Error("create_user returned no row"); + return row.id as string; + }, + + async createAgent(ownerId, name, id) { + const [row] = await sql` + select ${sch}.create_agent(${ownerId}, ${name}, ${id ?? null}) as id + `; + if (!row) throw new Error("create_agent returned no row"); + return row.id as string; + }, + + async createGroup(spaceId, name, id) { + const [row] = await sql` + select ${sch}.create_group(${spaceId}, ${name}, ${id ?? null}) as id + `; + if (!row) throw new Error("create_group returned no row"); + return row.id as string; + }, + + async getPrincipal(id) { + const [row] = await sql`select * from ${sch}.get_principal(${id})`; + return row ? mapPrincipal(row) : null; + }, + + async addPrincipalToSpace(spaceId, principalId, admin = false) { + await sql`select ${sch}.add_principal_to_space(${spaceId}, ${principalId}, ${admin})`; + }, + + async removePrincipalFromSpace(spaceId, principalId) { + const [row] = await sql` + select ${sch}.remove_principal_from_space(${spaceId}, ${principalId}) as removed + `; + return Boolean(row?.removed); + }, + + async addGroupMember(spaceId, groupId, memberId, admin = false) { + await sql`select ${sch}.add_group_member(${spaceId}, ${groupId}, ${memberId}, ${admin})`; + }, + + async removeGroupMember(spaceId, groupId, memberId) { + const [row] = await sql` + select ${sch}.remove_group_member(${spaceId}, ${groupId}, ${memberId}) as removed + `; + return Boolean(row?.removed); + }, + + async grantTreeAccess(spaceId, principalId, treePath, access) { + await sql` + select ${sch}.grant_tree_access(${spaceId}, ${principalId}, ${treePath}::ltree, ${access}) + `; + }, + + async removeTreeAccessGrant(spaceId, principalId, treePath) { + const [row] = await sql` + select ${sch}.remove_tree_access_grant(${spaceId}, ${principalId}, ${treePath}::ltree) as removed + `; + return Boolean(row?.removed); + }, + + async buildTreeAccess(memberId, spaceId) { + const [row] = await sql` + select ${sch}.build_tree_access(${memberId}, ${spaceId}) as ta + `; + return (row?.ta as TreeAccess) ?? []; + }, + + async createApiKey(memberId, name, opts) { + const lookupId = generateLookupId(); + const secret = generateSecret(); + const secretHash = hashApiKeySecret(secret); + const [row] = await sql` + select ${sch}.create_api_key( + ${memberId}, ${lookupId}, ${secretHash}, ${name}, ${opts?.expiresAt ?? null} + ) as id + `; + if (!row) throw new Error("create_api_key returned no row"); + return { id: row.id as string, lookupId, secret }; + }, + + async validateApiKey(lookupId, secret) { + const secretHash = hashApiKeySecret(secret); + const [row] = await sql` + select member_id, api_key_id + from ${sch}.validate_api_key(${lookupId}, ${secretHash}) + `; + if (!row) return null; + return { + memberId: row.member_id as string, + apiKeyId: row.api_key_id as string, + }; + }, + + async withTransaction(fn: (db: CoreDB) => Promise): Promise { + return sql.begin((tx) => + fn(createCoreDB(tx as unknown as Sql, schema)), + ) as Promise; + }, + }; + + return db; +} diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts new file mode 100644 index 0000000..c4bb981 --- /dev/null +++ b/packages/engine/core/index.ts @@ -0,0 +1,17 @@ +export { + formatApiKey, + generateLookupId, + generateSecret, + hashApiKeySecret, + parseApiKey, +} from "./api-key"; +export { type CoreDB, createCoreDB } from "./db"; +export type { + AccessLevel, + CreatedApiKey, + Principal, + PrincipalKind, + Space, + TreeAccess, + ValidatedApiKey, +} from "./types"; diff --git a/packages/engine/core/types.ts b/packages/engine/core/types.ts new file mode 100644 index 0000000..a556db9 --- /dev/null +++ b/packages/engine/core/types.ts @@ -0,0 +1,55 @@ +/** + * Types for the core control-plane TS layer. + * + * This layer is intentionally thin: every method calls a core SQL function + * (see packages/database/core/migrate/idempotent/*.sql) and never queries the + * core tables directly. + */ + +export type PrincipalKind = "u" | "g" | "a"; + +/** Access levels stored in core.tree_access: 1 = read, 2 = write, 3 = owner. */ +export type AccessLevel = 1 | 2 | 3; + +export interface Space { + id: string; + slug: string; + name: string; + language: string; + createdAt: Date; + updatedAt: Date | null; +} + +export interface Principal { + id: string; + kind: PrincipalKind; + name: string; + ownerId: string | null; + spaceId: string | null; + createdAt: Date; + updatedAt: Date | null; +} + +/** + * The effective access set for a member in a space, as produced by + * core.build_tree_access and consumed verbatim by the space data-plane + * functions (search_memory, get_memory, …). Kept in the on-the-wire snake_case + * shape because it is passed straight through to those functions as jsonb. + */ +export type TreeAccess = { tree_path: string; access: number }[]; + +export interface CreatedApiKey { + /** The api_key row id. */ + id: string; + /** The lookup id (goes in the key string, used for the indexed lookup). */ + lookupId: string; + /** The plaintext secret — returned once; only its sha256 hash is stored. */ + secret: string; +} + +export interface ValidatedApiKey { + /** The principal (user or agent) the key belongs to. */ + memberId: string; + /** The api_key row id. */ + apiKeyId: string; +} diff --git a/packages/engine/index.ts b/packages/engine/index.ts index e16b89c..ea402b8 100644 --- a/packages/engine/index.ts +++ b/packages/engine/index.ts @@ -1,4 +1,9 @@ // Main exports + +// New core control-plane layer (targets the `core` schema via SQL functions). +// Namespaced to avoid clashing with the legacy formatApiKey/parseApiKey above +// during the migration; consumers use core.createCoreDB, core.parseApiKey, etc. +export * as core from "./core"; export { type CreateEngineDBOptions, createEngineDB, diff --git a/packages/engine/package.json b/packages/engine/package.json index 5e214a9..39f044f 100644 --- a/packages/engine/package.json +++ b/packages/engine/package.json @@ -4,6 +4,8 @@ "private": true, "type": "module", "dependencies": { - "@pydantic/logfire-node": "^0.13.1" + "@memory.build/database": "workspace:*", + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9" } } From 2a3f4d0c3b7708aa951dadac93dc30954034770e Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 14:50:38 +0200 Subject: [PATCH 031/156] feat(engine): add space data-plane TS layer (createSpaceDB) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thin TS layer over the existing space SQL functions, in a new packages/engine/space module (exported as the `space` namespace). Each method calls a function and passes the treeAccess set (from core.buildTreeAccess) for access enforcement — no table queries, no RLS. - createSpaceDB(sql, schema): createMemory/getMemory/patchMemory/ deleteMemory, moveTree/copyTree/deleteTree/countTree/listTree, and search/hybridSearch (BM25 + vector + RRF). Uses postgres.js. - search takes a caller-supplied embedding (vec) — generation stays decoupled, as before; bm25 builds to_bm25query(text, index_name text). - jsonb args use sql.json (passing JSON.stringify double-encodes to a jsonb string scalar; a raw JS array would be sent as a Postgres array — both break jsonb_to_recordset). halfvec via a [..]::halfvec literal. Not wired into the server yet (Phase 4 switch). 7 integration tests green on local PG18 (CRUD, access enforcement, bm25/vector/hybrid search, tree ops); full check (1037 unit) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/engine/index.ts | 7 +- packages/engine/space/db.integration.test.ts | 148 +++++++++++ packages/engine/space/db.ts | 247 +++++++++++++++++++ packages/engine/space/index.ts | 13 + packages/engine/space/types.ts | 92 +++++++ 5 files changed, 504 insertions(+), 3 deletions(-) create mode 100644 packages/engine/space/db.integration.test.ts create mode 100644 packages/engine/space/db.ts create mode 100644 packages/engine/space/index.ts create mode 100644 packages/engine/space/types.ts diff --git a/packages/engine/index.ts b/packages/engine/index.ts index ea402b8..4433bda 100644 --- a/packages/engine/index.ts +++ b/packages/engine/index.ts @@ -1,8 +1,8 @@ // Main exports -// New core control-plane layer (targets the `core` schema via SQL functions). -// Namespaced to avoid clashing with the legacy formatApiKey/parseApiKey above -// during the migration; consumers use core.createCoreDB, core.parseApiKey, etc. +// New core control-plane + space data-plane layers (target the core / me_ +// schemas via SQL functions). Namespaced to avoid clashing with the legacy flat +// exports below during the migration: core.createCoreDB, space.createSpaceDB, etc. export * as core from "./core"; export { type CreateEngineDBOptions, @@ -11,6 +11,7 @@ export { } from "./db"; // Re-export migrate module export * from "./migrate"; +export * as space from "./space"; // Type exports export { type ApiKey, diff --git a/packages/engine/space/db.integration.test.ts b/packages/engine/space/db.integration.test.ts new file mode 100644 index 0000000..2deebef --- /dev/null +++ b/packages/engine/space/db.integration.test.ts @@ -0,0 +1,148 @@ +// Integration tests for the space data-plane TS layer (createSpaceDB). +// +// Provisions a throwaway metest_ schema via migrateSpace (small embedding +// dims for speed) and exercises the wrappers against the real SQL functions. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/engine/space/db.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateSpace } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { createSpaceDB, type SpaceDB } from "./db"; +import type { TreeAccess } from "./types"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +const randomSlug = () => { + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let s = ""; + for (const b of bytes) s += ALPHABET[b % 36]; + return s; +}; + +// Full owner access at "work"; all test memories live under work.* +const FULL: TreeAccess = [{ tree_path: "work", access: 3 }]; +const READONLY: TreeAccess = [{ tree_path: "work", access: 1 }]; + +let sql: Sql; +let schema: string; +let db: SpaceDB; + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + const slug = randomSlug(); + schema = `metest_${slug}`; + await migrateSpace(sql, { slug, schema, embeddingDimensions: 4 }); + db = createSpaceDB(sql, schema); +}); + +afterAll(async () => { + if (schema) await sql.unsafe(`drop schema if exists ${schema} cascade`); + await sql.end(); +}); + +/** Directly set a memory's embedding (simulating the worker). */ +async function setEmbedding(id: string, vec: number[]): Promise { + await sql.unsafe( + `update ${schema}.memory set embedding = $1::halfvec where id = $2`, + [`[${vec.join(",")}]`, id], + ); +} + +test("createMemory + getMemory round-trips", async () => { + const id = await db.createMemory(FULL, { + tree: "work.note", + content: "hello world", + meta: { kind: "note" }, + }); + const m = await db.getMemory(FULL, id); + expect(m?.id).toBe(id); + expect(m?.tree).toBe("work.note"); + expect(m?.content).toBe("hello world"); + expect(m?.meta).toEqual({ kind: "note" }); + expect(m?.hasEmbedding).toBe(false); +}); + +test("access is enforced by the tree_access argument", async () => { + // create requires write (>=2): read-only access is rejected + await expect( + db.createMemory(READONLY, { tree: "work.x", content: "nope" }), + ).rejects.toThrow(); + + // a memory is invisible to a tree_access set that doesn't cover its path + const id = await db.createMemory(FULL, { + tree: "work.secret", + content: "shh", + }); + const other: TreeAccess = [{ tree_path: "other", access: 3 }]; + expect(await db.getMemory(other, id)).toBeNull(); +}); + +test("patchMemory updates fields; deleteMemory removes", async () => { + const id = await db.createMemory(FULL, { + tree: "work.p", + content: "before", + }); + expect(await db.patchMemory(FULL, id, { content: "after" })).toBe(true); + expect((await db.getMemory(FULL, id))?.content).toBe("after"); + + expect(await db.deleteMemory(FULL, id)).toBe(true); + expect(await db.getMemory(FULL, id)).toBeNull(); +}); + +test("bm25 search ranks by full-text relevance", async () => { + await db.createMemory(FULL, { + tree: "work.a", + content: "the quick brown fox", + }); + await db.createMemory(FULL, { tree: "work.b", content: "lorem ipsum dolor" }); + + const results = await db.search(FULL, { bm25: "fox", limit: 5 }); + expect(results.length).toBeGreaterThanOrEqual(1); + expect(results[0]?.content).toContain("fox"); +}); + +test("vector search ranks by embedding similarity", async () => { + const near = await db.createMemory(FULL, { + tree: "work.v1", + content: "near", + }); + const far = await db.createMemory(FULL, { tree: "work.v2", content: "far" }); + await setEmbedding(near, [1, 0, 0, 0]); + await setEmbedding(far, [0, 1, 0, 0]); + + const results = await db.search(FULL, { vec: [1, 0, 0, 0], limit: 5 }); + expect(results[0]?.id).toBe(near); +}); + +test("hybridSearch fuses bm25 + vector", async () => { + const id = await db.createMemory(FULL, { + tree: "work.h", + content: "hybrid pineapple", + }); + await setEmbedding(id, [0, 0, 1, 0]); + + const results = await db.hybridSearch(FULL, { + bm25: "pineapple", + vec: [0, 0, 1, 0], + limit: 5, + }); + expect(results.some((r) => r.id === id)).toBe(true); +}); + +test("moveTree, countTree, listTree", async () => { + await db.createMemory(FULL, { tree: "work.src.one", content: "1" }); + await db.createMemory(FULL, { tree: "work.src.two", content: "2" }); + + expect(await db.countTree(FULL, { tree: "work.src" }, 1)).toBe(2); + + const moved = await db.moveTree(FULL, "work.src", "work.dst"); + expect(moved).toBe(2); + expect(await db.countTree(FULL, { tree: "work.src" }, 1)).toBe(0); + expect(await db.countTree(FULL, { tree: "work.dst" }, 1)).toBe(2); + + const listed = await db.listTree(FULL, "work.dst.*"); + expect(listed.some((e) => e.tree === "work.dst")).toBe(true); +}); diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts new file mode 100644 index 0000000..feb487d --- /dev/null +++ b/packages/engine/space/db.ts @@ -0,0 +1,247 @@ +import type { Sql } from "postgres"; +import type { AccessLevel } from "../core/types"; +import type { + CreateMemoryParams, + HybridSearchOptions, + Memory, + MemoryPatch, + SearchOptions, + SearchResultItem, + TreeAccess, + TreeListEntry, +} from "./types"; + +/** + * The space data-plane layer for one space schema (me_). + * + * Thin wrappers over the space SQL functions — each method calls a function and + * passes the `treeAccess` set (from core.buildTreeAccess) for access enforcement. + * No table queries in TS; no RLS (access is the jsonb argument). + */ +export interface SpaceDB { + createMemory( + treeAccess: TreeAccess, + params: CreateMemoryParams, + ): Promise; + getMemory(treeAccess: TreeAccess, id: string): Promise; + patchMemory( + treeAccess: TreeAccess, + id: string, + patch: MemoryPatch, + ): Promise; + deleteMemory(treeAccess: TreeAccess, id: string): Promise; + + moveTree( + treeAccess: TreeAccess, + src: string, + dst: string, + dryRun?: boolean, + ): Promise; + copyTree( + treeAccess: TreeAccess, + src: string, + dst: string, + dryRun?: boolean, + ): Promise; + deleteTree( + treeAccess: TreeAccess, + tree: string, + dryRun?: boolean, + ): Promise; + countTree( + treeAccess: TreeAccess, + query: { tree?: string; lquery?: string; ltxtquery?: string }, + access: AccessLevel, + ): Promise; + listTree(treeAccess: TreeAccess, lquery: string): Promise; + + search( + treeAccess: TreeAccess, + options?: SearchOptions, + ): Promise; + hybridSearch( + treeAccess: TreeAccess, + options: HybridSearchOptions, + ): Promise; +} + +function mapMemory(row: Record): Memory { + return { + id: row.id as string, + tree: row.tree as string, + meta: (row.meta as Record) ?? {}, + temporal: (row.temporal as string | null) ?? null, + content: row.content as string, + hasEmbedding: Boolean(row.has_embedding), + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapSearchItem(row: Record): SearchResultItem { + return { ...mapMemory(row), score: Number(row.score) }; +} + +export function createSpaceDB(sql: Sql, schema: string): SpaceDB { + const sch = sql(schema); + const bm25Index = `${schema}.memory_content_bm25_idx`; + + /** + * jsonb param fragment (null-safe). Uses sql.json so postgres.js serializes + * the value as json — passing a pre-stringified string double-encodes it into + * a jsonb string scalar, and passing a raw JS array would be sent as a Postgres + * array; both break jsonb_to_recordset in the SQL functions. + */ + const jb = (v: unknown) => + v === null || v === undefined + ? sql`null::jsonb` + : sql`${sql.json(v as never)}::jsonb`; + + /** bm25query fragment from a query string (or null). */ + const bm25 = (q: string | undefined) => + q === undefined + ? sql`null::bm25query` + : sql`to_bm25query(${q}::text, ${bm25Index}::text)`; + + /** halfvec fragment from an embedding (or null). */ + const halfvec = (v: number[] | undefined) => + v === undefined ? sql`null::halfvec` : sql`${`[${v.join(",")}]`}::halfvec`; + + return { + async createMemory(treeAccess, p) { + const [row] = await sql` + select ${sch}.create_memory( + ${jb(treeAccess)}, + ${p.tree}::ltree, + ${p.content}, + ${p.id ?? null}, + ${jb(p.meta)}, + ${p.temporal ?? null}::tstzrange + ) as id`; + if (!row) throw new Error("create_memory returned no row"); + return row.id as string; + }, + + async getMemory(treeAccess, id) { + const [row] = await sql` + select id, tree::text as tree, meta, temporal::text as temporal, + content, has_embedding, created_at, updated_at + from ${sch}.get_memory(${jb(treeAccess)}, ${id})`; + return row ? mapMemory(row) : null; + }, + + async patchMemory(treeAccess, id, patch) { + const obj: Record = {}; + if (patch.tree !== undefined) obj.tree = patch.tree; + if (patch.meta !== undefined) obj.meta = patch.meta; + if (patch.temporal !== undefined) obj.temporal = patch.temporal; + if (patch.content !== undefined) obj.content = patch.content; + const [row] = await sql` + select ${sch}.patch_memory(${jb(treeAccess)}, ${id}, ${jb(obj)}) as ok`; + return Boolean(row?.ok); + }, + + async deleteMemory(treeAccess, id) { + const [row] = await sql` + select ${sch}.delete_memory(${jb(treeAccess)}, ${id}) as ok`; + return Boolean(row?.ok); + }, + + async moveTree(treeAccess, src, dst, dryRun = false) { + const [row] = await sql` + select ${sch}.move_tree(${jb(treeAccess)}, ${src}::ltree, ${dst}::ltree, ${dryRun}) as n`; + return Number(row?.n); + }, + + async copyTree(treeAccess, src, dst, dryRun = false) { + const [row] = await sql` + select ${sch}.copy_tree(${jb(treeAccess)}, ${src}::ltree, ${dst}::ltree, ${dryRun}) as n`; + return Number(row?.n); + }, + + async deleteTree(treeAccess, tree, dryRun = false) { + const [row] = await sql` + select ${sch}.delete_tree(${jb(treeAccess)}, ${tree}::ltree, ${dryRun}) as n`; + return Number(row?.n); + }, + + async countTree(treeAccess, query, access) { + let row: { n?: unknown } | undefined; + if (query.tree !== undefined) { + [row] = await sql` + select ${sch}.count_tree(${jb(treeAccess)}, ${query.tree}::ltree, ${access}) as n`; + } else if (query.lquery !== undefined) { + [row] = await sql` + select ${sch}.count_tree(${jb(treeAccess)}, ${query.lquery}::lquery, ${access}) as n`; + } else if (query.ltxtquery !== undefined) { + [row] = await sql` + select ${sch}.count_tree(${jb(treeAccess)}, ${query.ltxtquery}::ltxtquery, ${access}) as n`; + } else { + throw new Error("countTree requires one of tree / lquery / ltxtquery"); + } + return Number(row?.n); + }, + + async listTree(treeAccess, lquery) { + const rows = await sql` + select tree::text as tree, count + from ${sch}.list_tree(${jb(treeAccess)}, ${lquery}::lquery)`; + return rows.map((r) => ({ + tree: r.tree as string, + count: Number(r.count), + })); + }, + + async search(treeAccess, options = {}) { + const o = options; + const rows = await sql` + select id, meta, tree::text as tree, temporal::text as temporal, + content, has_embedding, created_at, updated_at, score + from ${sch}.search_memory( + ${jb(treeAccess)}, + ${bm25(o.bm25)}, + ${halfvec(o.vec)}, + ${o.maxVecDist ?? null}, + ${o.ltree ?? null}::ltree, + ${o.lquery ?? null}::lquery, + ${o.ltxtquery ?? null}::ltxtquery, + ${jb(o.metaContains)}, + ${o.temporalWithin ?? null}::tstzrange, + ${o.temporalOverlaps ?? null}::tstzrange, + ${o.temporalBefore ?? null}::timestamptz, + ${o.temporalAfter ?? null}::timestamptz, + ${o.regexp ?? null}, + ${o.limit ?? 10} + )`; + return rows.map(mapSearchItem); + }, + + async hybridSearch(treeAccess, options) { + const o = options; + const rows = await sql` + select id, meta, tree::text as tree, temporal::text as temporal, + content, has_embedding, created_at, updated_at, score + from ${sch}.hybrid_search_memory( + ${jb(treeAccess)}, + ${bm25(o.bm25)}, + ${halfvec(o.vec)}, + ${o.maxVecDist ?? null}, + ${o.ltree ?? null}::ltree, + ${o.lquery ?? null}::lquery, + ${o.ltxtquery ?? null}::ltxtquery, + ${jb(o.metaContains)}, + ${o.temporalWithin ?? null}::tstzrange, + ${o.temporalOverlaps ?? null}::tstzrange, + ${o.temporalBefore ?? null}::timestamptz, + ${o.temporalAfter ?? null}::timestamptz, + ${o.regexp ?? null}, + ${o.k ?? 60.0}, + ${o.candidateLimit ?? 30}, + ${o.fulltextWeight ?? 1.0}, + ${o.semanticWeight ?? 1.0}, + ${o.limit ?? 10} + )`; + return rows.map(mapSearchItem); + }, + }; +} diff --git a/packages/engine/space/index.ts b/packages/engine/space/index.ts new file mode 100644 index 0000000..f7a23d9 --- /dev/null +++ b/packages/engine/space/index.ts @@ -0,0 +1,13 @@ +export { createSpaceDB, type SpaceDB } from "./db"; +export type { + CreateMemoryParams, + HybridSearchOptions, + Memory, + MemoryFilters, + MemoryPatch, + SearchOptions, + SearchResultItem, + TemporalRange, + TreeAccess, + TreeListEntry, +} from "./types"; diff --git a/packages/engine/space/types.ts b/packages/engine/space/types.ts new file mode 100644 index 0000000..70ed472 --- /dev/null +++ b/packages/engine/space/types.ts @@ -0,0 +1,92 @@ +/** + * Types for the space data-plane TS layer. + * + * Thin wrappers over the space SQL functions (packages/database/space/migrate/ + * idempotent/*.sql). Every method takes a `treeAccess` set — the jsonb produced + * by core.buildTreeAccess — which the SQL functions use to enforce access. + */ + +import type { TreeAccess } from "../core/types"; + +export type { TreeAccess }; + +/** tstzrange rendered as its text form, e.g. "[2024-01-01,2024-01-02)". */ +export type TemporalRange = string; + +export interface Memory { + id: string; + tree: string; + meta: Record; + temporal: TemporalRange | null; + content: string; + hasEmbedding: boolean; + createdAt: Date; + updatedAt: Date | null; +} + +export interface SearchResultItem extends Memory { + score: number; +} + +export interface CreateMemoryParams { + tree: string; + content: string; + id?: string; + meta?: Record; + temporal?: TemporalRange; +} + +export interface MemoryPatch { + tree?: string; + meta?: Record; + temporal?: TemporalRange | null; + content?: string; +} + +/** Filters shared by search (and the count/list tree helpers where relevant). */ +export interface MemoryFilters { + /** ancestor-or-self match: only memories at/under this path. */ + ltree?: string; + /** ltree lquery pattern. */ + lquery?: string; + /** ltree full-text ltxtquery. */ + ltxtquery?: string; + /** meta @> this object. */ + metaContains?: Record; + temporalWithin?: TemporalRange; + temporalOverlaps?: TemporalRange; + temporalBefore?: string; + temporalAfter?: string; + /** case-insensitive regexp on content (must be combined with another filter). */ + regexp?: string; +} + +export interface SearchOptions extends MemoryFilters { + /** BM25 full-text query string. Mutually exclusive with `vec`. */ + bm25?: string; + /** Pre-computed query embedding. Mutually exclusive with `bm25`. */ + vec?: number[]; + /** Max cosine distance (only with `vec`). */ + maxVecDist?: number; + limit?: number; +} + +export interface HybridSearchOptions extends MemoryFilters { + /** BM25 full-text query string (required). */ + bm25: string; + /** Pre-computed query embedding (required). */ + vec: number[]; + maxVecDist?: number; + /** RRF constant (default 60). */ + k?: number; + /** Per-arm candidate pool size (default 30). */ + candidateLimit?: number; + fulltextWeight?: number; + semanticWeight?: number; + limit?: number; +} + +export interface TreeListEntry { + tree: string; + count: number; +} From 20a8178265b8b991387efd9d68c6a18c09305385 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 15:29:11 +0200 Subject: [PATCH 032/156] refactor(engine): rename core/space DB layers to *Store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createCoreDB/CoreDB and createSpaceDB/SpaceDB read like they create or migrate the database; they actually build a typed interface that calls the SQL functions over an existing connection. Rename to coreStore/ CoreStore and spaceStore/SpaceStore. ("Store", not "Client" — Client is already the HTTP API client in packages/client.) The upcoming auth layer will follow as authStore. Pure rename; no behavior change. Typecheck, lint, and the core+space integration tests (14) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/engine/core/db.integration.test.ts | 8 ++++---- packages/engine/core/db.ts | 12 ++++++------ packages/engine/core/index.ts | 2 +- packages/engine/index.ts | 2 +- packages/engine/space/db.integration.test.ts | 8 ++++---- packages/engine/space/db.ts | 4 ++-- packages/engine/space/index.ts | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/engine/core/db.integration.test.ts b/packages/engine/core/db.integration.test.ts index ddbfe3f..5a27c04 100644 --- a/packages/engine/core/db.integration.test.ts +++ b/packages/engine/core/db.integration.test.ts @@ -1,4 +1,4 @@ -// Integration tests for the core control-plane TS layer (createCoreDB). +// Integration tests for the core control-plane TS layer (coreStore). // // Provisions a throwaway `core_test_` schema via migrateCore and exercises // the thin wrappers against the real SQL functions. Run with a database, e.g.: @@ -8,7 +8,7 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; import { migrateCore } from "@memory.build/database"; import postgres, { type Sql } from "postgres"; import { formatApiKey, parseApiKey } from "./api-key"; -import { type CoreDB, createCoreDB } from "./db"; +import { type CoreStore, coreStore } from "./db"; const URL = process.env.TEST_DATABASE_URL ?? @@ -26,13 +26,13 @@ const randomSlug = () => randomFrom(12); let sql: Sql; let schema: string; -let db: CoreDB; +let db: CoreStore; beforeAll(async () => { sql = postgres(URL, { onnotice: () => {} }); schema = randomCoreSchema(); await migrateCore(sql, { schema }); - db = createCoreDB(sql, schema); + db = coreStore(sql, schema); }); afterAll(async () => { diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index 0217e35..4dcecd0 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -18,7 +18,7 @@ import type { * packages/database/core/migrate/idempotent/*.sql; none query core tables * directly. Access enforcement and multi-table logic live in the SQL. */ -export interface CoreDB { +export interface CoreStore { createSpace(slug: string, name: string, language?: string): Promise; getSpace(slug: string): Promise; @@ -75,7 +75,7 @@ export interface CoreDB { ): Promise; /** Run operations atomically against the same transaction. */ - withTransaction(fn: (db: CoreDB) => Promise): Promise; + withTransaction(fn: (db: CoreStore) => Promise): Promise; } function mapSpace(row: Record): Space { @@ -101,10 +101,10 @@ function mapPrincipal(row: Record): Principal { }; } -export function createCoreDB(sql: Sql, schema: string = CORE_SCHEMA): CoreDB { +export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { const sch = sql(schema); // escaped schema identifier reused across queries - const db: CoreDB = { + const db: CoreStore = { async createSpace(slug, name, language) { const [row] = await sql` select ${sch}.create_space(${slug}, ${name}, ${language ?? null}) as id @@ -213,9 +213,9 @@ export function createCoreDB(sql: Sql, schema: string = CORE_SCHEMA): CoreDB { }; }, - async withTransaction(fn: (db: CoreDB) => Promise): Promise { + async withTransaction(fn: (db: CoreStore) => Promise): Promise { return sql.begin((tx) => - fn(createCoreDB(tx as unknown as Sql, schema)), + fn(coreStore(tx as unknown as Sql, schema)), ) as Promise; }, }; diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts index c4bb981..b96b5d3 100644 --- a/packages/engine/core/index.ts +++ b/packages/engine/core/index.ts @@ -5,7 +5,7 @@ export { hashApiKeySecret, parseApiKey, } from "./api-key"; -export { type CoreDB, createCoreDB } from "./db"; +export { type CoreStore, coreStore } from "./db"; export type { AccessLevel, CreatedApiKey, diff --git a/packages/engine/index.ts b/packages/engine/index.ts index 4433bda..f2ca310 100644 --- a/packages/engine/index.ts +++ b/packages/engine/index.ts @@ -2,7 +2,7 @@ // New core control-plane + space data-plane layers (target the core / me_ // schemas via SQL functions). Namespaced to avoid clashing with the legacy flat -// exports below during the migration: core.createCoreDB, space.createSpaceDB, etc. +// exports below during the migration: core.coreStore, space.spaceStore, etc. export * as core from "./core"; export { type CreateEngineDBOptions, diff --git a/packages/engine/space/db.integration.test.ts b/packages/engine/space/db.integration.test.ts index 2deebef..45f6991 100644 --- a/packages/engine/space/db.integration.test.ts +++ b/packages/engine/space/db.integration.test.ts @@ -1,4 +1,4 @@ -// Integration tests for the space data-plane TS layer (createSpaceDB). +// Integration tests for the space data-plane TS layer (spaceStore). // // Provisions a throwaway metest_ schema via migrateSpace (small embedding // dims for speed) and exercises the wrappers against the real SQL functions. @@ -7,7 +7,7 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; import { migrateSpace } from "@memory.build/database"; import postgres, { type Sql } from "postgres"; -import { createSpaceDB, type SpaceDB } from "./db"; +import { type SpaceStore, spaceStore } from "./db"; import type { TreeAccess } from "./types"; const URL = @@ -28,14 +28,14 @@ const READONLY: TreeAccess = [{ tree_path: "work", access: 1 }]; let sql: Sql; let schema: string; -let db: SpaceDB; +let db: SpaceStore; beforeAll(async () => { sql = postgres(URL, { onnotice: () => {} }); const slug = randomSlug(); schema = `metest_${slug}`; await migrateSpace(sql, { slug, schema, embeddingDimensions: 4 }); - db = createSpaceDB(sql, schema); + db = spaceStore(sql, schema); }); afterAll(async () => { diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts index feb487d..fdb9e3f 100644 --- a/packages/engine/space/db.ts +++ b/packages/engine/space/db.ts @@ -18,7 +18,7 @@ import type { * passes the `treeAccess` set (from core.buildTreeAccess) for access enforcement. * No table queries in TS; no RLS (access is the jsonb argument). */ -export interface SpaceDB { +export interface SpaceStore { createMemory( treeAccess: TreeAccess, params: CreateMemoryParams, @@ -82,7 +82,7 @@ function mapSearchItem(row: Record): SearchResultItem { return { ...mapMemory(row), score: Number(row.score) }; } -export function createSpaceDB(sql: Sql, schema: string): SpaceDB { +export function spaceStore(sql: Sql, schema: string): SpaceStore { const sch = sql(schema); const bm25Index = `${schema}.memory_content_bm25_idx`; diff --git a/packages/engine/space/index.ts b/packages/engine/space/index.ts index f7a23d9..c210533 100644 --- a/packages/engine/space/index.ts +++ b/packages/engine/space/index.ts @@ -1,4 +1,4 @@ -export { createSpaceDB, type SpaceDB } from "./db"; +export { type SpaceStore, spaceStore } from "./db"; export type { CreateMemoryParams, HybridSearchOptions, From 0e74e5605cfaf4f546b6087409bbb8858c69d9b5 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 16:07:54 +0200 Subject: [PATCH 033/156] feat(database): add auth schema SQL functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the auth schema's control-plane functions, mirroring the core/space convention (logic in SQL; security invoker; search_path set). The TS layer (next) only calls these. - user: create_user (email_verified/image), get_user, get_user_by_email (compares as citext so the lookup stays case-insensitive). - session: create_session, validate_session (join + expiry), delete_session, delete_sessions_by_user (revoke-all), cleanup_expired_sessions. - account (oauth links, login-only — no tokens, no email; BA-shaped): upsert_account, get_account_by_provider (resolve user by (provider_id, account_id), which prevents cross-provider takeover), get_accounts_by_user, unlink_account. - device flow: create_device_auth, get_device_by_{user_code,oauth_state}, authorize_device, deny_device, poll_device (resolves the poll state machine in one call: expired/slow_down/denied/pending/authorized), delete_device, delete_expired_devices. 24 integration tests green (local PG18). TODO.md notes deferred search_path tightening (+ possibly moving extensions off public). Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 20 ++ .../auth/migrate/idempotent/001_user.sql | 60 ++++++ .../auth/migrate/idempotent/002_session.sql | 86 ++++++++ .../auth/migrate/idempotent/003_account.sql | 80 ++++++++ .../migrate/idempotent/004_device_auth.sql | 190 ++++++++++++++++++ .../auth/migrate/migrate.integration.test.ts | 181 ++++++++++++++++- packages/database/auth/migrate/migrate.ts | 22 ++ 7 files changed, 638 insertions(+), 1 deletion(-) create mode 100644 packages/database/auth/migrate/idempotent/001_user.sql create mode 100644 packages/database/auth/migrate/idempotent/002_session.sql create mode 100644 packages/database/auth/migrate/idempotent/003_account.sql create mode 100644 packages/database/auth/migrate/idempotent/004_device_auth.sql diff --git a/TODO.md b/TODO.md index a233f89..536e758 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,26 @@ later is cheap if distribution returns. They're separate packages, so sharing with them needs a dev-only package (or fold them in during the postgres.js rollout). +## Harden `search_path` on SQL functions (+ maybe move extensions off `public`) + +All the schema SQL functions currently set +`search_path to pg_catalog, {{schema}}, public, pg_temp`. They can be tightened +(every object reference is already schema-qualified, and none create temp +objects): + +- [ ] Auth (and likely core/space) data functions → `pg_catalog, public`: drop + `{{schema}}` (nothing unqualified) and `pg_temp` (so a temp object can never + shadow). The SECURITY DEFINER `update_updated_at` trigger fn can go all the + way to `search_path = ''` — it only uses `pg_catalog.now()` + the NEW record. +- [ ] `public` only has to stay because of `citext` (the `users.email` column + + the `_email::citext` compare; its `=` operator can't be cleanly + schema-qualified in `a = b`). Consider installing extensions + (`citext`, and engine's `ltree`/`vector`/`pg_textsearch`) into a dedicated + `extensions` schema instead of `public`; then the path becomes + `pg_catalog, extensions` and `public` drops out entirely. This touches the + migrate bootstrap (`ensureExtension` installs `with schema public`) — decide + holistically before changing the function `search_path`s. + ## Consolidate the migration runner logic - [x] Done — the shared machinery lives in `packages/database/migrate/kit.ts`: diff --git a/packages/database/auth/migrate/idempotent/001_user.sql b/packages/database/auth/migrate/idempotent/001_user.sql new file mode 100644 index 0000000..f681f1c --- /dev/null +++ b/packages/database/auth/migrate/idempotent/001_user.sql @@ -0,0 +1,60 @@ +------------------------------------------------------------------------------- +-- create_user +-- email_verified is set from the provider's verified-email flag by the caller. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_user +( _email text +, _name text +, _email_verified bool default false +, _image text default null +) +returns uuid +as $func$ + insert into {{schema}}.users (email, name, email_verified, image) + values (_email, _name, coalesce(_email_verified, false), _image) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_user +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_user(_id uuid) +returns table +( id uuid +, email text +, name text +, email_verified bool +, image text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select u.id, u.email::text, u.name, u.email_verified, u.image, u.created_at, u.updated_at + from {{schema}}.users u + where u.id = _id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_user_by_email (citext column -> case-insensitive match) +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_user_by_email(_email text) +returns table +( id uuid +, email text +, name text +, email_verified bool +, image text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select u.id, u.email::text, u.name, u.email_verified, u.image, u.created_at, u.updated_at + from {{schema}}.users u + where u.email = _email::citext -- compare as citext (case-insensitive); a text param would force text=text +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/idempotent/002_session.sql b/packages/database/auth/migrate/idempotent/002_session.sql new file mode 100644 index 0000000..b61d1a3 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/002_session.sql @@ -0,0 +1,86 @@ +------------------------------------------------------------------------------- +-- create_session +-- The caller generates the token and passes its hash (sha256); the plaintext +-- token is never stored. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_session +( _user_id uuid +, _token_hash bytea +, _expires_at timestamptz +) +returns uuid +as $func$ + insert into {{schema}}.sessions (user_id, token_hash, expires_at) + values (_user_id, _token_hash, _expires_at) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- validate_session +-- Looks up an unexpired session by token hash and returns the session + its +-- user. No rows if missing or expired. +------------------------------------------------------------------------------- +create or replace function {{schema}}.validate_session(_token_hash bytea) +returns table +( session_id uuid +, user_id uuid +, email text +, name text +, expires_at timestamptz +) +as $func$ + select s.id, u.id, u.email::text, u.name, s.expires_at + from {{schema}}.sessions s + inner join {{schema}}.users u on (u.id = s.user_id) + where s.token_hash = _token_hash + and s.expires_at > now() +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_session +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_session(_id uuid) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.sessions where id = _id returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_sessions_by_user (revoke all) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_sessions_by_user(_user_id uuid) +returns bigint +as $func$ + with d as + ( + delete from {{schema}}.sessions where user_id = _user_id returning 1 + ) + select count(*) from d +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- cleanup_expired_sessions (cron) +------------------------------------------------------------------------------- +create or replace function {{schema}}.cleanup_expired_sessions() +returns bigint +as $func$ + with d as + ( + delete from {{schema}}.sessions where expires_at <= now() returning 1 + ) + select count(*) from d +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/idempotent/003_account.sql b/packages/database/auth/migrate/idempotent/003_account.sql new file mode 100644 index 0000000..e2b01ba --- /dev/null +++ b/packages/database/auth/migrate/idempotent/003_account.sql @@ -0,0 +1,80 @@ +------------------------------------------------------------------------------- +-- upsert_account +-- Links an OAuth provider account to a user. Login-only: no tokens stored, and +-- no email (the verified email lives on users.email — better-auth-shaped). +-- The (provider_id, account_id) pair is the stable identity key. +------------------------------------------------------------------------------- +create or replace function {{schema}}.upsert_account +( _user_id uuid +, _provider_id text +, _account_id text +) +returns uuid +as $func$ + insert into {{schema}}.accounts (user_id, provider_id, account_id) + values (_user_id, _provider_id, _account_id) + on conflict (provider_id, account_id) do update set + user_id = excluded.user_id -- updated_at maintained by the before-update trigger + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_account_by_provider +-- The login lookup: resolves the owning user from the provider account id. +-- Resolving by (provider_id, account_id) — NOT by email — is what prevents +-- account-takeover via a different provider asserting the same address. +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_account_by_provider +( _provider_id text +, _account_id text +) +returns table +( id uuid +, user_id uuid +, provider_id text +, account_id text +) +as $func$ + select a.id, a.user_id, a.provider_id, a.account_id + from {{schema}}.accounts a + where a.provider_id = _provider_id + and a.account_id = _account_id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_accounts_by_user +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_accounts_by_user(_user_id uuid) +returns table +( id uuid +, user_id uuid +, provider_id text +, account_id text +) +as $func$ + select a.id, a.user_id, a.provider_id, a.account_id + from {{schema}}.accounts a + where a.user_id = _user_id + order by a.created_at +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- unlink_account +------------------------------------------------------------------------------- +create or replace function {{schema}}.unlink_account(_id uuid) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.accounts where id = _id returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/idempotent/004_device_auth.sql b/packages/database/auth/migrate/idempotent/004_device_auth.sql new file mode 100644 index 0000000..28a1d12 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/004_device_auth.sql @@ -0,0 +1,190 @@ +------------------------------------------------------------------------------- +-- create_device_auth (OAuth 2.0 device flow) +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_device_auth +( _device_code text +, _user_code text +, _provider text +, _oauth_state text +, _expires_at timestamptz +) +returns void +as $func$ + insert into {{schema}}.device_authorization + (device_code, user_code, provider, oauth_state, expires_at) + values + (_device_code, _user_code, _provider, _oauth_state, _expires_at) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +-- shared return shape for the device lookups (unexpired rows only) +------------------------------------------------------------------------------- +-- get_device_by_user_code (browser code entry; caller normalizes the code) +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_device_by_user_code(_user_code text) +returns table +( device_code text +, user_code text +, provider text +, oauth_state text +, expires_at timestamptz +, last_poll timestamptz +, user_id uuid +, denied bool +, created_at timestamptz +) +as $func$ + select d.device_code, d.user_code, d.provider, d.oauth_state, d.expires_at, + d.last_poll, d.user_id, d.denied, d.created_at + from {{schema}}.device_authorization d + where d.user_code = _user_code and d.expires_at > now() +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_device_by_oauth_state (OAuth callback) +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_device_by_oauth_state(_oauth_state text) +returns table +( device_code text +, user_code text +, provider text +, oauth_state text +, expires_at timestamptz +, last_poll timestamptz +, user_id uuid +, denied bool +, created_at timestamptz +) +as $func$ + select d.device_code, d.user_code, d.provider, d.oauth_state, d.expires_at, + d.last_poll, d.user_id, d.denied, d.created_at + from {{schema}}.device_authorization d + where d.oauth_state = _oauth_state and d.expires_at > now() +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- authorize_device (callback success → bind the user) +------------------------------------------------------------------------------- +create or replace function {{schema}}.authorize_device(_device_code text, _user_id uuid) +returns bool +as $func$ + with u as + ( + update {{schema}}.device_authorization + set user_id = _user_id + where device_code = _device_code + and expires_at > now() + and user_id is null + and denied = false + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- deny_device +------------------------------------------------------------------------------- +create or replace function {{schema}}.deny_device(_device_code text) +returns bool +as $func$ + with u as + ( + update {{schema}}.device_authorization + set denied = true + where device_code = _device_code + and expires_at > now() + and user_id is null + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- poll_device (CLI polling — resolves the device-flow state in one call) +-- Returns a status, plus the bound user when authorized: +-- 'expired' — no unexpired device with this code +-- 'slow_down' — polled within _min_interval_secs (last_poll NOT advanced) +-- 'denied' — the user denied the request +-- 'pending' — not yet authorized +-- 'authorized' — bound to user_id (caller then mints a session + deletes it) +-- Subsumes the old get_device_by_device_code + update_device_last_poll. +------------------------------------------------------------------------------- +create or replace function {{schema}}.poll_device +( _device_code text +, _min_interval_secs double precision default 5 +) +returns table (status text, user_id uuid) +as $func$ +declare + _d record; +begin + select d.* into _d + from {{schema}}.device_authorization d + where d.device_code = _device_code and d.expires_at > now(); + + if not found then + return query select 'expired'::text, null::uuid; + return; + end if; + + -- rate limit: polled too recently -> slow_down, without advancing last_poll + if _d.last_poll is not null + and extract(epoch from now() - _d.last_poll) < _min_interval_secs then + return query select 'slow_down'::text, null::uuid; + return; + end if; + + update {{schema}}.device_authorization + set last_poll = now() + where device_code = _device_code; + + if _d.denied then + return query select 'denied'::text, null::uuid; + elsif _d.user_id is not null then + return query select 'authorized'::text, _d.user_id; + else + return query select 'pending'::text, null::uuid; + end if; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_device (cleanup after completion) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_device(_device_code text) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.device_authorization where device_code = _device_code returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_expired_devices (cron) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_expired_devices() +returns bigint +as $func$ + with d as + ( + delete from {{schema}}.device_authorization where expires_at <= now() returning 1 + ) + select count(*) from d +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/migrate.integration.test.ts b/packages/database/auth/migrate/migrate.integration.test.ts index c603d36..1e635ac 100644 --- a/packages/database/auth/migrate/migrate.integration.test.ts +++ b/packages/database/auth/migrate/migrate.integration.test.ts @@ -43,7 +43,19 @@ const EXPECTED_MIGRATIONS = [ "005_verifications", ]; -const EXPECTED_FUNCTIONS = ["update_updated_at"]; +const EXPECTED_FUNCTIONS = [ + "update_updated_at", + "create_user", + "get_user", + "get_user_by_email", + "create_session", + "validate_session", + "upsert_account", + "get_account_by_provider", + "create_device_auth", + "authorize_device", + "poll_device", +]; // The auth schema deliberately requires only citext — not the engine extensions. const REQUIRED_EXTENSIONS = ["citext"]; @@ -218,6 +230,173 @@ describe("schema constraints enforce", () => { }); }); +describe("auth functions", () => { + const email = () => `fn_${crypto.randomUUID().slice(0, 8)}@example.com`; + + test("create_user + get_user + get_user_by_email (citext)", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const e = email(); + const [u] = await sql.unsafe( + `select ${s}.create_user($1, $2, $3) as id`, + [e, "Alice", true], + ); + const id = u?.id as string; + + const [byId] = await sql.unsafe(`select * from ${s}.get_user($1)`, [id]); + expect(byId?.id).toBe(id); + expect(byId?.email).toBe(e); + expect(byId?.email_verified).toBe(true); + + // citext: lookup is case-insensitive + const [byEmail] = await sql.unsafe( + `select * from ${s}.get_user_by_email($1)`, + [e.toUpperCase()], + ); + expect(byEmail?.id).toBe(id); + }); + }); + + test("create_session + validate_session (valid + expired)", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const [u] = await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + email(), + "Bob", + ]); + const userId = u?.id as string; + + await sql.unsafe( + `select ${s}.create_session($1, $2::bytea, now() + interval '1 day')`, + [userId, "\\xabcd"], + ); + const valid = await sql.unsafe( + `select * from ${s}.validate_session($1::bytea)`, + ["\\xabcd"], + ); + expect(valid.length).toBe(1); + expect(valid[0]?.user_id).toBe(userId); + + await sql.unsafe( + `select ${s}.create_session($1, $2::bytea, now() - interval '1 second')`, + [userId, "\\xbeef"], + ); + const expired = await sql.unsafe( + `select * from ${s}.validate_session($1::bytea)`, + ["\\xbeef"], + ); + expect(expired.length).toBe(0); + }); + }); + + test("upsert_account + get_account_by_provider (login lookup, idempotent)", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const [u] = await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + email(), + "Carol", + ]); + const userId = u?.id as string; + const acct = crypto.randomUUID(); + + await sql.unsafe(`select ${s}.upsert_account($1, 'github', $2)`, [ + userId, + acct, + ]); + const found = await sql.unsafe( + `select * from ${s}.get_account_by_provider('github', $1)`, + [acct], + ); + expect(found[0]?.user_id).toBe(userId); + + // re-upsert the same (provider, account) stays one row + await sql.unsafe(`select ${s}.upsert_account($1, 'github', $2)`, [ + userId, + acct, + ]); + const [n] = await sql.unsafe( + `select count(*)::int as n from ${s}.accounts where provider_id='github' and account_id=$1`, + [acct], + ); + expect(n?.n).toBe(1); + + const none = await sql.unsafe( + `select * from ${s}.get_account_by_provider('github', $1)`, + [crypto.randomUUID()], + ); + expect(none.length).toBe(0); + }); + }); + + test("device flow: create → lookups → poll_device → authorize", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const [u] = await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + email(), + "Dave", + ]); + const userId = u?.id as string; + + const deviceCode = crypto.randomUUID(); + const userCode = "ABCD-2345"; + const oauthState = crypto.randomUUID(); + await sql.unsafe( + `select ${s}.create_device_auth($1, $2, 'google', $3, now() + interval '15 min')`, + [deviceCode, userCode, oauthState], + ); + + const byState = await sql.unsafe( + `select * from ${s}.get_device_by_oauth_state($1)`, + [oauthState], + ); + expect(byState[0]?.device_code).toBe(deviceCode); + const byUserCode = await sql.unsafe( + `select * from ${s}.get_device_by_user_code($1)`, + [userCode], + ); + expect(byUserCode[0]?.device_code).toBe(deviceCode); + + // poll before authorization → pending (interval 0 bypasses rate limit) + const [p1] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + deviceCode, + ]); + expect(p1?.status).toBe("pending"); + expect(p1?.user_id).toBeNull(); + + // immediate re-poll with the default interval → slow_down + const [sd] = await sql.unsafe(`select * from ${s}.poll_device($1)`, [ + deviceCode, + ]); + expect(sd?.status).toBe("slow_down"); + + // authorize binds the user; a second authorize is a no-op + const [a] = await sql.unsafe( + `select ${s}.authorize_device($1, $2) as ok`, + [deviceCode, userId], + ); + expect(a?.ok).toBe(true); + const [a2] = await sql.unsafe( + `select ${s}.authorize_device($1, $2) as ok`, + [deviceCode, userId], + ); + expect(a2?.ok).toBe(false); + + // poll now resolves to authorized + the bound user + const [p2] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + deviceCode, + ]); + expect(p2?.status).toBe("authorized"); + expect(p2?.user_id).toBe(userId); + + // unknown / expired device code → expired + const [ex] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + crypto.randomUUID(), + ]); + expect(ex?.status).toBe("expired"); + }); + }); +}); + describe("cascade + trigger behavior", () => { test("deleting a user cascades to accounts and sessions", async () => { await withTestAuth(sql, {}, async (auth) => { diff --git a/packages/database/auth/migrate/migrate.ts b/packages/database/auth/migrate/migrate.ts index d3e4014..545f2ad 100644 --- a/packages/database/auth/migrate/migrate.ts +++ b/packages/database/auth/migrate/migrate.ts @@ -16,6 +16,12 @@ import { } from "../../migrate/kit"; import { AUTH_SCHEMA_VERSION } from "../version"; import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; +import idempotent001 from "./idempotent/001_user.sql" with { type: "text" }; +import idempotent002 from "./idempotent/002_session.sql" with { type: "text" }; +import idempotent003 from "./idempotent/003_account.sql" with { type: "text" }; +import idempotent004 from "./idempotent/004_device_auth.sql" with { + type: "text", +}; import incremental001 from "./incremental/001_users.sql" with { type: "text" }; import incremental002 from "./incremental/002_accounts.sql" with { type: "text", @@ -66,6 +72,22 @@ const incrementals: Migration[] = [ const idempotents: Migration[] = [ { name: "000_update", file: "idempotent/000_update.sql", sql: idempotent000 }, + { name: "001_user", file: "idempotent/001_user.sql", sql: idempotent001 }, + { + name: "002_session", + file: "idempotent/002_session.sql", + sql: idempotent002, + }, + { + name: "003_account", + file: "idempotent/003_account.sql", + sql: idempotent003, + }, + { + name: "004_device_auth", + file: "idempotent/004_device_auth.sql", + sql: idempotent004, + }, ]; /** From 884053184ab5e2999434ac0bc6a11b6951677311 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 16:14:41 +0200 Subject: [PATCH 034/156] feat(auth): add authStore runtime layer (packages/auth) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New @memory.build/auth package: a thin TS layer over the auth schema SQL functions (postgres.js). Every method calls a function; token generation/hashing is the only TS-side logic. - authStore(sql, schema): users (create/get/getByEmail), sessions (createSession mints a token + returns it once, validateSession, delete, deleteSessionsByUser, cleanupExpired), accounts (upsertAccount, getAccountByProvider, getAccountsByUser, unlink), device flow (createDeviceAuth generates the codes, getDeviceByUserCode [normalizes input], getDeviceByOAuthState, pollDevice, authorize/deny, delete/deleteExpired), and withTransaction. - token.ts: session token gen + sha256 hash (bytea), and device-code / user-code / oauth-state generators — logically identical to the old hash.ts + device-flow.ts (same 32B/16B sizes, same ambiguity-free user-code charset, 15-min expiry, 5s poll interval), just consolidated onto one base64url helper. Not wired into the server yet (Phase 4). 6 integration tests green on local PG18; typecheck + lint clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- bun.lock | 10 + packages/auth/db.integration.test.ts | 131 ++++++++++++ packages/auth/db.ts | 286 +++++++++++++++++++++++++++ packages/auth/index.ts | 21 ++ packages/auth/package.json | 10 + packages/auth/token.ts | 62 ++++++ packages/auth/types.ts | 66 +++++++ 7 files changed, 586 insertions(+) create mode 100644 packages/auth/db.integration.test.ts create mode 100644 packages/auth/db.ts create mode 100644 packages/auth/index.ts create mode 100644 packages/auth/package.json create mode 100644 packages/auth/token.ts create mode 100644 packages/auth/types.ts diff --git a/bun.lock b/bun.lock index 73094dc..a473b1f 100644 --- a/bun.lock +++ b/bun.lock @@ -19,6 +19,14 @@ "@pydantic/logfire-node": "^0.13.1", }, }, + "packages/auth": { + "name": "@memory.build/auth", + "version": "0.0.0", + "dependencies": { + "@memory.build/database": "workspace:*", + "postgres": "^3.4.9", + }, + }, "packages/cli": { "name": "@memory.build/cli", "version": "0.2.6", @@ -377,6 +385,8 @@ "@memory.build/accounts": ["@memory.build/accounts@workspace:packages/accounts"], + "@memory.build/auth": ["@memory.build/auth@workspace:packages/auth"], + "@memory.build/cli": ["@memory.build/cli@workspace:packages/cli"], "@memory.build/client": ["@memory.build/client@workspace:packages/client"], diff --git a/packages/auth/db.integration.test.ts b/packages/auth/db.integration.test.ts new file mode 100644 index 0000000..5fbc1b4 --- /dev/null +++ b/packages/auth/db.integration.test.ts @@ -0,0 +1,131 @@ +// Integration tests for the auth runtime layer (authStore). +// +// Provisions a throwaway auth_test_ schema via migrateAuth and exercises +// the wrappers against the real SQL functions. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/auth/db.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateAuth } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { type AuthStore, authStore } from "./db"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +const randomAuthSchema = () => { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += ALPHABET[b % 36]; + return `auth_test_${s}`; +}; +const email = () => `fn_${crypto.randomUUID().slice(0, 8)}@example.com`; + +let sql: Sql; +let schema: string; +let db: AuthStore; + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + schema = randomAuthSchema(); + await migrateAuth(sql, { schema }); + db = authStore(sql, schema); +}); + +afterAll(async () => { + if (schema) await sql.unsafe(`drop schema if exists ${schema} cascade`); + await sql.end(); +}); + +test("createUser + getUser + getUserByEmail (case-insensitive)", async () => { + const e = email(); + const id = await db.createUser(e, "Alice", { emailVerified: true }); + + const byId = await db.getUser(id); + expect(byId?.id).toBe(id); + expect(byId?.email).toBe(e); + expect(byId?.emailVerified).toBe(true); + + const byEmail = await db.getUserByEmail(e.toUpperCase()); + expect(byEmail?.id).toBe(id); + + expect(await db.getUserByEmail(email())).toBeNull(); +}); + +test("createSession returns a token that validateSession accepts", async () => { + const id = await db.createUser(email(), "Bob"); + const { sessionId, token } = await db.createSession(id); + expect(token.length).toBeGreaterThan(20); + + const v = await db.validateSession(token); + expect(v?.sessionId).toBe(sessionId); + expect(v?.userId).toBe(id); + + // wrong token → null + expect(await db.validateSession("not-a-real-token")).toBeNull(); + + // after delete → null + expect(await db.deleteSession(sessionId)).toBe(true); + expect(await db.validateSession(token)).toBeNull(); +}); + +test("deleteSessionsByUser revokes all of a user's sessions", async () => { + const id = await db.createUser(email(), "Carol"); + const a = await db.createSession(id); + const b = await db.createSession(id); + + expect(await db.deleteSessionsByUser(id)).toBe(2); + expect(await db.validateSession(a.token)).toBeNull(); + expect(await db.validateSession(b.token)).toBeNull(); +}); + +test("upsertAccount + getAccountByProvider", async () => { + const id = await db.createUser(email(), "Dave"); + const acct = crypto.randomUUID(); + + await db.upsertAccount(id, "github", acct); + const found = await db.getAccountByProvider("github", acct); + expect(found?.userId).toBe(id); + expect(found?.providerId).toBe("github"); + + // idempotent + await db.upsertAccount(id, "github", acct); + expect((await db.getAccountsByUser(id)).length).toBe(1); + + expect( + await db.getAccountByProvider("github", crypto.randomUUID()), + ).toBeNull(); +}); + +test("device flow: create → lookup (normalized code) → poll → authorize", async () => { + const id = await db.createUser(email(), "Erin"); + const { deviceCode, userCode, oauthState } = + await db.createDeviceAuth("google"); + + // user_code lookup tolerates lowercase / missing hyphen + const denorm = userCode.toLowerCase().replace("-", ""); + const byUserCode = await db.getDeviceByUserCode(denorm); + expect(byUserCode?.deviceCode).toBe(deviceCode); + + const byState = await db.getDeviceByOAuthState(oauthState); + expect(byState?.deviceCode).toBe(deviceCode); + + // pending → authorize → authorized + expect((await db.pollDevice(deviceCode, 0)).status).toBe("pending"); + expect(await db.authorizeDevice(deviceCode, id)).toBe(true); + const poll = await db.pollDevice(deviceCode, 0); + expect(poll.status).toBe("authorized"); + expect(poll.userId).toBe(id); +}); + +test("withTransaction rolls back on error", async () => { + const e = email(); + await expect( + db.withTransaction(async (tx) => { + await tx.createUser(e, "Rollback"); + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + expect(await db.getUserByEmail(e)).toBeNull(); +}); diff --git a/packages/auth/db.ts b/packages/auth/db.ts new file mode 100644 index 0000000..509d75a --- /dev/null +++ b/packages/auth/db.ts @@ -0,0 +1,286 @@ +import { AUTH_SCHEMA } from "@memory.build/database"; +import type { Sql } from "postgres"; +import { + DEVICE_CODE_EXPIRY_SECONDS, + generateDeviceCode, + generateOAuthState, + generateSessionToken, + generateUserCode, + hashSessionToken, + normalizeUserCode, +} from "./token"; +import type { + Account, + CreatedDeviceAuth, + CreatedSession, + CreateUserOptions, + DevicePollResult, + DevicePollStatus, + OAuthProvider, + User, + ValidatedSession, +} from "./types"; + +const SESSION_EXPIRY_DAYS = 30; + +/** + * The auth control-plane data layer. + * + * Thin wrappers over the auth schema SQL functions; every method calls a + * function (none query auth tables directly). Token generation/hashing is the + * only TS-side logic (the DB stores only hashes). + */ +export interface AuthStore { + createUser( + email: string, + name: string, + opts?: CreateUserOptions, + ): Promise; + getUser(id: string): Promise; + getUserByEmail(email: string): Promise; + + /** Mint a session; returns the one-time raw token (only its hash is stored). */ + createSession( + userId: string, + opts?: { expiresInDays?: number }, + ): Promise; + validateSession(token: string): Promise; + deleteSession(id: string): Promise; + deleteSessionsByUser(userId: string): Promise; + cleanupExpiredSessions(): Promise; + + upsertAccount( + userId: string, + providerId: OAuthProvider, + accountId: string, + ): Promise; + getAccountByProvider( + providerId: OAuthProvider, + accountId: string, + ): Promise; + getAccountsByUser(userId: string): Promise; + unlinkAccount(id: string): Promise; + + /** Start a device flow; generates the codes and returns them. */ + createDeviceAuth(provider: OAuthProvider): Promise; + getDeviceByUserCode(userCode: string): Promise; + getDeviceByOAuthState(oauthState: string): Promise; + /** Resolve the poll state machine in one call (see core poll_device). */ + pollDevice( + deviceCode: string, + minIntervalSecs?: number, + ): Promise; + authorizeDevice(deviceCode: string, userId: string): Promise; + denyDevice(deviceCode: string): Promise; + deleteDevice(deviceCode: string): Promise; + deleteExpiredDevices(): Promise; + + withTransaction(fn: (db: AuthStore) => Promise): Promise; +} + +/** A device authorization row (the get_device_by_* lookups). */ +export interface DeviceAuthRow { + deviceCode: string; + userCode: string; + provider: OAuthProvider; + oauthState: string; + expiresAt: Date; + lastPoll: Date | null; + userId: string | null; + denied: boolean; + createdAt: Date; +} + +function mapUser(row: Record): User { + return { + id: row.id as string, + email: row.email as string, + name: row.name as string, + emailVerified: Boolean(row.email_verified), + image: (row.image as string | null) ?? null, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapAccount(row: Record): Account { + return { + id: row.id as string, + userId: row.user_id as string, + providerId: row.provider_id as OAuthProvider, + accountId: row.account_id as string, + }; +} + +function mapDevice(row: Record): DeviceAuthRow { + return { + deviceCode: row.device_code as string, + userCode: row.user_code as string, + provider: row.provider as OAuthProvider, + oauthState: row.oauth_state as string, + expiresAt: row.expires_at as Date, + lastPoll: (row.last_poll as Date | null) ?? null, + userId: (row.user_id as string | null) ?? null, + denied: Boolean(row.denied), + createdAt: row.created_at as Date, + }; +} + +export function authStore(sql: Sql, schema: string = AUTH_SCHEMA): AuthStore { + const sch = sql(schema); + + const db: AuthStore = { + async createUser(email, name, opts) { + const [row] = await sql` + select ${sch}.create_user( + ${email}, ${name}, ${opts?.emailVerified ?? false}, ${opts?.image ?? null} + ) as id`; + if (!row) throw new Error("create_user returned no row"); + return row.id as string; + }, + + async getUser(id) { + const [row] = await sql`select * from ${sch}.get_user(${id})`; + return row ? mapUser(row) : null; + }, + + async getUserByEmail(email) { + const [row] = await sql`select * from ${sch}.get_user_by_email(${email})`; + return row ? mapUser(row) : null; + }, + + async createSession(userId, opts) { + const token = generateSessionToken(); + const tokenHash = hashSessionToken(token); + const days = opts?.expiresInDays ?? SESSION_EXPIRY_DAYS; + const expiresAt = new Date(Date.now() + days * 24 * 60 * 60 * 1000); + const [row] = await sql` + select ${sch}.create_session(${userId}, ${tokenHash}, ${expiresAt}) as id`; + if (!row) throw new Error("create_session returned no row"); + return { sessionId: row.id as string, token }; + }, + + async validateSession(token) { + const tokenHash = hashSessionToken(token); + const [row] = await sql` + select * from ${sch}.validate_session(${tokenHash})`; + if (!row) return null; + return { + sessionId: row.session_id as string, + userId: row.user_id as string, + email: row.email as string, + name: row.name as string, + expiresAt: row.expires_at as Date, + }; + }, + + async deleteSession(id) { + const [row] = await sql`select ${sch}.delete_session(${id}) as ok`; + return Boolean(row?.ok); + }, + + async deleteSessionsByUser(userId) { + const [row] = await sql` + select ${sch}.delete_sessions_by_user(${userId}) as n`; + return Number(row?.n); + }, + + async cleanupExpiredSessions() { + const [row] = await sql`select ${sch}.cleanup_expired_sessions() as n`; + return Number(row?.n); + }, + + async upsertAccount(userId, providerId, accountId) { + const [row] = await sql` + select ${sch}.upsert_account(${userId}, ${providerId}, ${accountId}) as id`; + if (!row) throw new Error("upsert_account returned no row"); + return row.id as string; + }, + + async getAccountByProvider(providerId, accountId) { + const [row] = await sql` + select * from ${sch}.get_account_by_provider(${providerId}, ${accountId})`; + return row ? mapAccount(row) : null; + }, + + async getAccountsByUser(userId) { + const rows = await sql` + select * from ${sch}.get_accounts_by_user(${userId})`; + return rows.map(mapAccount); + }, + + async unlinkAccount(id) { + const [row] = await sql`select ${sch}.unlink_account(${id}) as ok`; + return Boolean(row?.ok); + }, + + async createDeviceAuth(provider) { + const deviceCode = generateDeviceCode(); + const userCode = generateUserCode(); + const oauthState = generateOAuthState(); + const expiresAt = new Date( + Date.now() + DEVICE_CODE_EXPIRY_SECONDS * 1000, + ); + await sql` + select ${sch}.create_device_auth( + ${deviceCode}, ${userCode}, ${provider}, ${oauthState}, ${expiresAt} + )`; + return { + deviceCode, + userCode, + oauthState, + expiresIn: DEVICE_CODE_EXPIRY_SECONDS, + }; + }, + + async getDeviceByUserCode(userCode) { + const [row] = await sql` + select * from ${sch}.get_device_by_user_code(${normalizeUserCode(userCode)})`; + return row ? mapDevice(row) : null; + }, + + async getDeviceByOAuthState(oauthState) { + const [row] = await sql` + select * from ${sch}.get_device_by_oauth_state(${oauthState})`; + return row ? mapDevice(row) : null; + }, + + async pollDevice(deviceCode, minIntervalSecs) { + const [row] = await sql` + select * from ${sch}.poll_device(${deviceCode}, ${minIntervalSecs ?? 5})`; + return { + status: (row?.status as DevicePollStatus) ?? "expired", + userId: (row?.user_id as string | null) ?? null, + }; + }, + + async authorizeDevice(deviceCode, userId) { + const [row] = await sql` + select ${sch}.authorize_device(${deviceCode}, ${userId}) as ok`; + return Boolean(row?.ok); + }, + + async denyDevice(deviceCode) { + const [row] = await sql`select ${sch}.deny_device(${deviceCode}) as ok`; + return Boolean(row?.ok); + }, + + async deleteDevice(deviceCode) { + const [row] = await sql`select ${sch}.delete_device(${deviceCode}) as ok`; + return Boolean(row?.ok); + }, + + async deleteExpiredDevices() { + const [row] = await sql`select ${sch}.delete_expired_devices() as n`; + return Number(row?.n); + }, + + async withTransaction(fn: (db: AuthStore) => Promise): Promise { + return sql.begin((tx) => + fn(authStore(tx as unknown as Sql, schema)), + ) as Promise; + }, + }; + + return db; +} diff --git a/packages/auth/index.ts b/packages/auth/index.ts new file mode 100644 index 0000000..d0441bd --- /dev/null +++ b/packages/auth/index.ts @@ -0,0 +1,21 @@ +export { type AuthStore, authStore, type DeviceAuthRow } from "./db"; +export { + DEVICE_CODE_EXPIRY_SECONDS, + generateDeviceCode, + generateOAuthState, + generateSessionToken, + generateUserCode, + hashSessionToken, + normalizeUserCode, +} from "./token"; +export type { + Account, + CreatedDeviceAuth, + CreatedSession, + CreateUserOptions, + DevicePollResult, + DevicePollStatus, + OAuthProvider, + User, + ValidatedSession, +} from "./types"; diff --git a/packages/auth/package.json b/packages/auth/package.json new file mode 100644 index 0000000..d5011b2 --- /dev/null +++ b/packages/auth/package.json @@ -0,0 +1,10 @@ +{ + "name": "@memory.build/auth", + "version": "0.0.0", + "private": true, + "type": "module", + "dependencies": { + "@memory.build/database": "workspace:*", + "postgres": "^3.4.9" + } +} diff --git a/packages/auth/token.ts b/packages/auth/token.ts new file mode 100644 index 0000000..3bbea94 --- /dev/null +++ b/packages/auth/token.ts @@ -0,0 +1,62 @@ +/** + * Token + device-code helpers for the auth layer. + * + * Session tokens are 256-bit CSPRNG values; we store sha256(token) (bytea) and + * look up by hash — entropy alone defeats offline preimage attacks, so a fast + * hash is sufficient and a DB read never yields usable bearer tokens. + */ + +const SESSION_TOKEN_BYTES = 32; +const DEVICE_CODE_BYTES = 32; +const OAUTH_STATE_BYTES = 16; + +/** User code: 8 chars, excluding ambiguous 0/O/1/I/L, shown as XXXX-XXXX. */ +const USER_CODE_CHARS = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"; + +/** Device authorization lifetime (15 minutes), in seconds. */ +export const DEVICE_CODE_EXPIRY_SECONDS = 15 * 60; + +function base64url(bytes: Uint8Array): string { + return btoa(String.fromCharCode(...bytes)) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=/g, ""); +} + +function randomBase64url(byteLength: number): string { + return base64url(crypto.getRandomValues(new Uint8Array(byteLength))); +} + +/** Generate a random 256-bit session token (base64url). */ +export function generateSessionToken(): string { + return randomBase64url(SESSION_TOKEN_BYTES); +} + +/** sha256(token) as raw bytes, for the `token_hash` bytea column. */ +export function hashSessionToken(token: string): Buffer { + return new Bun.CryptoHasher("sha256").update(token).digest(); +} + +/** Device flow: the CLI polling secret (base64url, 32 bytes). */ +export function generateDeviceCode(): string { + return randomBase64url(DEVICE_CODE_BYTES); +} + +/** Device flow: the OAuth `state` (CSRF binding, base64url, 16 bytes). */ +export function generateOAuthState(): string { + return randomBase64url(OAUTH_STATE_BYTES); +} + +/** Device flow: the human-entered code, formatted XXXX-XXXX. */ +export function generateUserCode(): string { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let code = ""; + for (const b of bytes) code += USER_CODE_CHARS[b % USER_CODE_CHARS.length]; + return `${code.slice(0, 4)}-${code.slice(4)}`; +} + +/** Normalize user-entered codes (uppercase, strip hyphens, re-hyphenate). */ +export function normalizeUserCode(input: string): string { + const c = input.toUpperCase().replace(/-/g, ""); + return `${c.slice(0, 4)}-${c.slice(4)}`; +} diff --git a/packages/auth/types.ts b/packages/auth/types.ts new file mode 100644 index 0000000..6bfadad --- /dev/null +++ b/packages/auth/types.ts @@ -0,0 +1,66 @@ +/** + * Types for the auth runtime layer (authStore). + * + * Thin wrappers over the auth schema SQL functions + * (packages/database/auth/migrate/idempotent/*.sql). No table queries in TS. + */ + +export type OAuthProvider = "google" | "github"; + +export interface User { + id: string; + email: string; + name: string; + emailVerified: boolean; + image: string | null; + createdAt: Date; + updatedAt: Date | null; +} + +export interface CreateUserOptions { + emailVerified?: boolean; + image?: string; +} + +/** What validate_session returns: the session plus its user. */ +export interface ValidatedSession { + sessionId: string; + userId: string; + email: string; + name: string; + expiresAt: Date; +} + +/** A freshly minted session — the raw token is returned once, only its hash is stored. */ +export interface CreatedSession { + sessionId: string; + token: string; +} + +export interface Account { + id: string; + userId: string; + providerId: OAuthProvider; + accountId: string; +} + +export interface CreatedDeviceAuth { + deviceCode: string; + userCode: string; + oauthState: string; + /** Seconds until the device authorization expires. */ + expiresIn: number; +} + +export type DevicePollStatus = + | "expired" + | "slow_down" + | "denied" + | "pending" + | "authorized"; + +export interface DevicePollResult { + status: DevicePollStatus; + /** Set only when status === "authorized". */ + userId: string | null; +} From 27d855ecaae5d4cbcd2ee411a3b6092129350589 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 16:19:23 +0200 Subject: [PATCH 035/156] refactor(database): add provisionSpace(tx) for caller-transaction provisioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract a provisionSpace(tx, options) that provisions + migrates a space schema within the CALLER's transaction — no own sql.begin, no advisory lock. PostgreSQL DDL is transactional, so a caller (Phase 3D's provisionUser) can create the me_ schema + the core/auth rows in one transaction and have a failure roll the schema back — no orphan schema, no cleanup code. migrateSpace keeps its behavior (own transaction + advisory lock) for the standalone re-migrate path; both now share a provisionAndRun body + resolveSchema validator. No lock in provisionSpace is safe: a freshly generated slug has no contender, and schema-name / space.slug uniqueness makes any race fail-and-roll-back. Guard test: provisionSpace inside an aborted transaction rolls the whole schema back (incl. bm25/hnsw index DDL); the success path commits a fully-migrated space. 21 space tests + full check (1049) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/database/space/index.ts | 6 +- .../space/migrate/migrate.integration.test.ts | 38 +++++- packages/database/space/migrate/migrate.ts | 120 +++++++++++------- 3 files changed, 113 insertions(+), 51 deletions(-) diff --git a/packages/database/space/index.ts b/packages/database/space/index.ts index 2ca99e8..f639b5b 100644 --- a/packages/database/space/index.ts +++ b/packages/database/space/index.ts @@ -1,5 +1,9 @@ export { bootstrapSpaceDatabase } from "./migrate/bootstrap"; -export { type MigrateSpaceOptions, migrateSpace } from "./migrate/migrate"; +export { + type MigrateSpaceOptions, + migrateSpace, + provisionSpace, +} from "./migrate/migrate"; export { isValidSlug, isValidSpaceSchema, diff --git a/packages/database/space/migrate/migrate.integration.test.ts b/packages/database/space/migrate/migrate.integration.test.ts index c46594a..d8da816 100644 --- a/packages/database/space/migrate/migrate.integration.test.ts +++ b/packages/database/space/migrate/migrate.integration.test.ts @@ -16,7 +16,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import type { Sql as SQL } from "postgres"; import { SPACE_SCHEMA_VERSION } from "../version"; import { bootstrapSpaceDatabase } from "./bootstrap"; -import { migrateSpace } from "./migrate"; +import { migrateSpace, provisionSpace } from "./migrate"; import { appliedMigrations, columnType, @@ -28,8 +28,10 @@ import { listIndexes, listTables, listTriggers, + randomSlug, schemaExists, TestSpace, + tableExists, withTestSpace, } from "./test-utils"; @@ -94,6 +96,40 @@ afterAll(async () => { await sql.end(); }); +describe("provisionSpace (caller-transaction, transactional DDL)", () => { + test("rolls back the whole schema when the caller's transaction aborts", async () => { + const slug = randomSlug(); + const schema = `metest_${slug}`; + await expect( + sql.begin(async (tx) => { + await provisionSpace(tx, { slug, schema, embeddingDimensions: 4 }); + // visible inside the transaction (schema + memory table created) + const [r] = + await tx`select to_regclass(${`${schema}.memory`}) is not null as present`; + expect(r?.present).toBe(true); + throw new Error("rollback"); + }), + ).rejects.toThrow("rollback"); + // schema + bm25/hnsw index DDL all rolled back — nothing left behind + expect(await schemaExists(sql, schema)).toBe(false); + }); + + test("commits a fully-migrated space when the transaction succeeds", async () => { + const slug = randomSlug(); + const schema = `metest_${slug}`; + try { + await sql.begin(async (tx) => { + await provisionSpace(tx, { slug, schema, embeddingDimensions: 4 }); + }); + expect(await schemaExists(sql, schema)).toBe(true); + expect(await tableExists(sql, schema, "memory")).toBe(true); + expect(await getSchemaVersion(sql, schema)).toBe(SPACE_SCHEMA_VERSION); + } finally { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + } + }); +}); + describe("provisioned space schema", () => { test("creates the space schema", async () => { expect(await schemaExists(sql, canonical.schema)).toBe(true); diff --git a/packages/database/space/migrate/migrate.ts b/packages/database/space/migrate/migrate.ts index 660e0d8..4c7836d 100644 --- a/packages/database/space/migrate/migrate.ts +++ b/packages/database/space/migrate/migrate.ts @@ -1,6 +1,6 @@ import { info, reportError, span } from "@pydantic/logfire-node"; import { semver } from "bun"; -import type { Sql as SQL } from "postgres"; +import type { ISql, Sql as SQL } from "postgres"; import { acquireAdvisoryLock, advisoryLockKey, @@ -90,6 +90,74 @@ interface NormalizedMigrateSpaceOptions { idleInTransactionSessionTimeout: string; } +/** Validate options and resolve the target schema name. */ +function resolveSchema(opts: NormalizedMigrateSpaceOptions): string { + if (!isValidSlug(opts.slug)) { + throw new Error( + `Invalid space slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, + ); + } + if (opts.schema !== undefined && !isValidSchemaName(opts.schema)) { + throw new Error( + `Invalid schema override: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, + ); + } + if (!semver.satisfies(opts.schemaVersion, "*")) { + throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); + } + return opts.schema ?? slugToSchema(opts.slug); +} + +/** Provision (if missing) then run all migrations, against a given transaction. */ +async function provisionAndRun( + tx: ISql, + schema: string, + opts: NormalizedMigrateSpaceOptions, +): Promise { + if (!(await doesSchemaExist(tx, schema))) { + await executeSqlFile(tx, template(provisionSql, { schema }), { + logSqlFiles: opts.logSqlFiles, + label: "space", + schema, + type: "provision", + dir: DIR, + file: "provision.sql", + }); + } + await runSchemaMigrations(tx, { + schema, + schemaVersion: opts.schemaVersion, + incrementals, + idempotents, + templateVars: templateVars(schema, opts), + label: "space", + dir: DIR, + logSqlFiles: opts.logSqlFiles, + }); +} + +/** + * Provision + migrate a space schema within the CALLER's transaction — no own + * transaction and no advisory lock. Because schema creation is transactional, + * the caller can compose this atomically with other writes (e.g. provisionUser + * creates the me_ schema + the core/auth rows in one transaction, so any + * failure rolls the schema back — no orphan, no cleanup). + * + * Use `migrateSpace` for the standalone re-migrate path: it owns its + * transaction + advisory lock to serialize concurrent migrators of an existing + * space. Skipping the lock here is safe — a freshly generated slug has no + * contender, and the schema-name / `space.slug` uniqueness makes any race + * fail-and-roll-back. + */ +export async function provisionSpace( + tx: ISql, + options: MigrateSpaceOptions, +): Promise { + const opts = normalizeMigrateSpaceOptions(options); + const schema = resolveSchema(opts); + await provisionAndRun(tx, schema, opts); +} + export async function migrateSpace( sql: SQL, options: MigrateSpaceOptions, @@ -101,20 +169,7 @@ export async function migrateSpace( attributes, callback: async () => { try { - if (!isValidSlug(opts.slug)) { - throw new Error( - `Invalid space slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, - ); - } - if (opts.schema !== undefined && !isValidSchemaName(opts.schema)) { - throw new Error( - `Invalid schema override: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, - ); - } - if (!semver.satisfies(opts.schemaVersion, "*")) { - throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); - } - const schema = opts.schema ?? slugToSchema(opts.slug); + const schema = resolveSchema(opts); const schemaAttributes = { ...attributes, "db.schema": schema }; const [key1, key2] = advisoryLockKey(`memory-space:schema:${schema}`); @@ -129,40 +184,7 @@ export async function migrateSpace( `Unable to acquire lock for space slug ${opts.slug} migrations.`, ); } - - if (!(await doesSchemaExist(tx, schema))) { - await span("space.migrate.provision", { - attributes: { - ...schemaAttributes, - "space.migration_file": "provision.sql", - "space.migration_type": "provision", - }, - callback: () => - executeSqlFile(tx, template(provisionSql, { schema }), { - logSqlFiles: opts.logSqlFiles, - label: "space", - schema, - type: "provision", - dir: DIR, - file: "provision.sql", - }), - }); - info("Space schema provisioned", schemaAttributes); - } - await span("space.migrate.run", { - attributes: schemaAttributes, - callback: () => - runSchemaMigrations(tx, { - schema, - schemaVersion: opts.schemaVersion, - incrementals, - idempotents, - templateVars: templateVars(schema, opts), - label: "space", - dir: DIR, - logSqlFiles: opts.logSqlFiles, - }), - }); + await provisionAndRun(tx, schema, opts); }); info("Space migrations completed", schemaAttributes); } catch (error) { From 308139c679cf5f31e841177d062613033c71e586 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 16:40:57 +0200 Subject: [PATCH 036/156] =?UTF-8?q?feat(server):=20add=20provisionUser=20?= =?UTF-8?q?=E2=80=94=20atomic=20first-login=20provisioning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit provisionUser(sql, { auth, core }, params) stands up a brand-new user in ONE transaction: auth.users + the OAuth account link, a core.principal sharing the SAME id, a default core.space + its me_ data schema (via provisionSpace(tx)), and the user's owner grant on the space root. Schema creation is transactional, so any failure rolls the whole thing back — no orphan schema. No API key is minted (keys are agent-only; humans reach the engine via their session). Runs over a single connection spanning auth+core+space (the one-pool model Phase 4 wires). Also: - generateSlug() added to packages/database/space/slug.ts. - ACCESS ({read,write,owner}) and ROOT_PATH ("" — the empty ltree, which is ancestor of all, so an owner grant there covers the whole space) added to core types, replacing magic 1/2/3 and "". - server gains @memory.build/auth, @memory.build/database, postgres deps. - TODO: lenient user-facing tree-path normalization (accept / and ., leading slash, "" or "/" for root) at the boundary; pick a display form. 2 integration tests (full provisioning + atomic rollback) + full check (1051) green. Not wired into the server yet (Phase 4). Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 19 +++ bun.lock | 3 + packages/database/space/index.ts | 1 + packages/database/space/slug.ts | 9 ++ packages/engine/core/index.ts | 1 + packages/engine/core/types.ts | 15 ++ packages/server/package.json | 3 + packages/server/provision.integration.test.ts | 133 ++++++++++++++++++ packages/server/provision.ts | 77 ++++++++++ 9 files changed, 261 insertions(+) create mode 100644 packages/server/provision.integration.test.ts create mode 100644 packages/server/provision.ts diff --git a/TODO.md b/TODO.md index 536e758..a874e96 100644 --- a/TODO.md +++ b/TODO.md @@ -51,6 +51,25 @@ objects): migrate bootstrap (`ensureExtension` installs `with schema public`) — decide holistically before changing the function `search_path`s. +## User-facing tree-path convention (lenient input → canonical ltree) + +Tree paths are stored as ltree (dot-separated; the root is the empty path, +exported as `core.ROOT_PATH`). Internally everything stays ltree-native (the +store layer, the SQL functions, `provisionUser`). At the **user-facing boundary** +(RPC handlers, CLI, MCP) we want lenient input normalized once to that canonical +form — the right convention is what's natural for users, not what ltree accepts. + +- [ ] Add a shared `normalizeTreePath(input): string` util (home: alongside the + slug helpers in `packages/database/space`, or a small `path.ts`). Rules: + split on `/[./]+/`, drop empty segments, validate each is a legal ltree + label, join with `.`. So `/foo/bar`, `foo/bar`, `foo.bar` → `foo.bar`; and + `""`, `/`, `.` → `""` (root). Use it in **every** user-facing entry point + so they behave identically. Wire in Phase 4 with the memory/grant RPC + + CLI + MCP. +- [ ] Decide the canonical **output/display** form (echoed in search results, + `grant list`, etc.): dot-style `work.projects` (matches current docs) vs + filesystem-style `/work/projects`. Input stays lenient; output is one form. + ## Consolidate the migration runner logic - [x] Done — the shared machinery lives in `packages/database/migrate/kit.ts`: diff --git a/bun.lock b/bun.lock index a473b1f..d0d3997 100644 --- a/bun.lock +++ b/bun.lock @@ -133,11 +133,14 @@ "version": "0.2.5", "dependencies": { "@memory.build/accounts": "workspace:*", + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", "@memory.build/engine": "workspace:*", "@memory.build/protocol": "workspace:*", "@memory.build/worker": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", "zod": "^4.0.0", }, }, diff --git a/packages/database/space/index.ts b/packages/database/space/index.ts index f639b5b..f1ed765 100644 --- a/packages/database/space/index.ts +++ b/packages/database/space/index.ts @@ -5,6 +5,7 @@ export { provisionSpace, } from "./migrate/migrate"; export { + generateSlug, isValidSlug, isValidSpaceSchema, schemaToSlug, diff --git a/packages/database/space/slug.ts b/packages/database/space/slug.ts index 067b323..557f7d1 100644 --- a/packages/database/space/slug.ts +++ b/packages/database/space/slug.ts @@ -1,5 +1,14 @@ const SPACE_SCHEMA_RE = /^me_[a-z0-9]{12}$/; const SLUG_RE = /^[a-z0-9]{12}$/; +const SLUG_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; + +/** Generate a random 12-char lowercase-alphanumeric space slug. */ +export function generateSlug(): string { + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let slug = ""; + for (const b of bytes) slug += SLUG_ALPHABET[b % 36]; + return slug; +} export function isValidSpaceSchema(name: string): boolean { return SPACE_SCHEMA_RE.test(name); diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts index b96b5d3..917c967 100644 --- a/packages/engine/core/index.ts +++ b/packages/engine/core/index.ts @@ -15,3 +15,4 @@ export type { TreeAccess, ValidatedApiKey, } from "./types"; +export { ACCESS, ROOT_PATH } from "./types"; diff --git a/packages/engine/core/types.ts b/packages/engine/core/types.ts index a556db9..1a36a65 100644 --- a/packages/engine/core/types.ts +++ b/packages/engine/core/types.ts @@ -11,6 +11,21 @@ export type PrincipalKind = "u" | "g" | "a"; /** Access levels stored in core.tree_access: 1 = read, 2 = write, 3 = owner. */ export type AccessLevel = 1 | 2 | 3; +/** Named tree-access levels — use instead of the raw 1/2/3. */ +export const ACCESS = { + read: 1, + write: 2, + owner: 3, +} as const satisfies Record; + +/** + * The root tree path: the empty ltree (`''`), which is the ancestor of every + * path — so a grant here covers the whole space. (ltree separates with `.`, not + * `/`, and its root is the empty path; `/` is not an ltree concept and is + * reserved for agent names like `user/agent`.) + */ +export const ROOT_PATH = ""; + export interface Space { id: string; slug: string; diff --git a/packages/server/package.json b/packages/server/package.json index f747f73..cab7bba 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -3,11 +3,14 @@ "version": "0.2.5", "dependencies": { "@memory.build/accounts": "workspace:*", + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", "@memory.build/engine": "workspace:*", "@memory.build/protocol": "workspace:*", "@memory.build/worker": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", "zod": "^4.0.0" }, "exports": { diff --git a/packages/server/provision.integration.test.ts b/packages/server/provision.integration.test.ts new file mode 100644 index 0000000..e65a1f1 --- /dev/null +++ b/packages/server/provision.integration.test.ts @@ -0,0 +1,133 @@ +// Integration test for first-login provisioning (provisionUser). +// +// Stands up auth + core schemas and bootstraps the space DB in one database, +// then provisions users through a single connection (the one-pool model the +// server consolidates to in Phase 4). +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/server/provision.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import { core as engineCore } from "@memory.build/engine"; +import postgres, { type Sql } from "postgres"; +import { provisionUser } from "./provision"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; +const email = () => `prov_${crypto.randomUUID().slice(0, 8)}@example.com`; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +async function schemaExists(name: string): Promise { + const [r] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${name} + ) as e`; + return Boolean(r?.e); +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); // extensions for me_ + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +test("provisions a new user: identity + principal + space + owner grant", async () => { + const e = email(); + const accountId = crypto.randomUUID(); + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { email: e, name: "Alice", provider: "github", accountId }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + + // auth.users + const user = await authStore(sql, authSchema).getUser(r.userId); + expect(user?.email).toBe(e); + + // oauth account link resolves back to the user + const acct = await authStore(sql, authSchema).getAccountByProvider( + "github", + accountId, + ); + expect(acct?.userId).toBe(r.userId); + + // core principal shares the same id + const principal = await engineCore + .coreStore(sql, coreSchema) + .getPrincipal(r.userId); + expect(principal?.kind).toBe("u"); + expect(principal?.id).toBe(r.userId); + + // space registered in core + its data schema exists + const space = await engineCore + .coreStore(sql, coreSchema) + .getSpace(r.spaceSlug); + expect(space?.id).toBe(r.spaceId); + expect(await schemaExists(`me_${r.spaceSlug}`)).toBe(true); + + // owner of the space root + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(r.userId, r.spaceId); + expect(ta).toContainEqual({ + tree_path: engineCore.ROOT_PATH, + access: engineCore.ACCESS.owner, + }); +}); + +test("is atomic: a failure rolls everything back", async () => { + const e = email(); + const a1 = crypto.randomUUID(); + const r1 = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { email: e, name: "Bob", provider: "github", accountId: a1 }, + ); + createdSpaceSchemas.push(`me_${r1.spaceSlug}`); + + // re-provisioning the same email fails (users.email is unique) — the whole + // transaction must roll back, leaving no trace of the second attempt. + const a2 = crypto.randomUUID(); + await expect( + provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { email: e, name: "Bob2", provider: "github", accountId: a2 }, + ), + ).rejects.toThrow(); + + // the second account link was rolled back + expect( + await authStore(sql, authSchema).getAccountByProvider("github", a2), + ).toBeNull(); +}); diff --git a/packages/server/provision.ts b/packages/server/provision.ts new file mode 100644 index 0000000..b3443d4 --- /dev/null +++ b/packages/server/provision.ts @@ -0,0 +1,77 @@ +import { authStore, type OAuthProvider } from "@memory.build/auth"; +import { generateSlug, provisionSpace } from "@memory.build/database"; +import { core as engineCore } from "@memory.build/engine"; +import type { Sql } from "postgres"; + +/** + * First-login provisioning. + * + * Atomically (one transaction) stands up everything a brand-new user needs: + * - auth.users row (the global identity) + the OAuth account link + * - core.principal (kind 'u') sharing the SAME id as auth.users + * - a default core.space + its me_ data schema (provisionSpace runs the + * schema DDL inside this transaction) + * - the user's owner grant on the space root + * + * Because schema creation is transactional, any failure rolls the whole thing + * back — no orphaned me_ schema, no cleanup code. No API key is minted: + * keys are agent-only; humans reach the engine via their session. + * + * Requires a single connection that can write the auth, core, and me_ + * schemas (the DB must already be bootstrapped with the required extensions). + */ +export interface ProvisionUserParams { + email: string; + /** Display name, stored on auth.users. */ + name: string; + provider: OAuthProvider; + /** The provider's stable account id (the OAuth `sub`). */ + accountId: string; + emailVerified?: boolean; + image?: string; + /** Name for the personal space (default "default"). */ + spaceName?: string; +} + +export interface ProvisionUserResult { + userId: string; + spaceId: string; + spaceSlug: string; +} + +export function provisionUser( + sql: Sql, + schemas: { auth: string; core: string }, + params: ProvisionUserParams, +): Promise { + const slug = generateSlug(); + + return sql.begin(async (tx) => { + const auth = authStore(tx as unknown as Sql, schemas.auth); + const core = engineCore.coreStore(tx as unknown as Sql, schemas.core); + + const userId = await auth.createUser(params.email, params.name, { + emailVerified: params.emailVerified, + image: params.image, + }); + await auth.upsertAccount(userId, params.provider, params.accountId); + + // The core principal shares the auth user id (one identity across schemas). + // Its globally-unique principal name is the email — the natural unique + // handle for a user (display name lives on auth.users.name). + await core.createUser(userId, params.email); + + const spaceId = await core.createSpace(slug, params.spaceName ?? "default"); + await provisionSpace(tx, { slug }); // creates the me_ data schema + await core.addPrincipalToSpace(spaceId, userId, true); + // owner of the root path → the user owns the whole space + await core.grantTreeAccess( + spaceId, + userId, + engineCore.ROOT_PATH, + engineCore.ACCESS.owner, + ); + + return { userId, spaceId, spaceSlug: slug }; + }) as Promise; +} From d1f434104720b5c4513d4253937cfadcdcffad31 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 17:07:02 +0200 Subject: [PATCH 037/156] feat(server): boot-migrate auth+core on a single postgres.js pool (Phase 4A) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce the new-model database pool and provision its schemas at startup, ahead of wiring the auth/memory paths onto them: - add a single postgres.js `db` pool (engineDatabaseUrl) — auth + core + per-space me_ all live in one database, one pool. The legacy Bun.SQL accountsSql/engineSql pools remain until Phase 5 removes the old code paths. - at boot, alongside the legacy bootstrap/migrations: bootstrapSpaceDatabase + migrateCore + migrateAuth on `db`. Close it on shutdown. Additive: the old login/memory paths are unchanged; the new pool and schemas are consumed as 4B+ come online. Typecheck/lint/full check green; boot sequence smoke-tested on one pool against local PG18. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/server/index.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/server/index.ts b/packages/server/index.ts index ead57ea..e322489 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -1,6 +1,11 @@ // packages/server/index.ts import { createAccountsDB } from "@memory.build/accounts"; import { migrate as migrateAccounts } from "@memory.build/accounts/migrate/runner"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; import type { EmbeddingConfig } from "@memory.build/embedding"; import { discoverEngineSchemas, @@ -14,6 +19,7 @@ import { } from "@memory.build/engine/ops/_tx"; import { WorkerPool } from "@memory.build/worker"; import { configure, info, reportError, span } from "@pydantic/logfire-node"; +import postgres from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import { embeddingConstants } from "./config"; import type { ServerContext } from "./context"; @@ -352,6 +358,17 @@ const workerEngineSql = new Bun.SQL(workerEngineDatabaseUrl, { connectionTimeout: workerEnginePoolConnectionTimeout, }); +// New-model pool (postgres.js): the auth + core control plane and the per-space +// me_ data schemas all live in one database, one pool. The legacy Bun.SQL +// accountsSql/engineSql pools above stay until Phase 5 removes the old paths. +const db = postgres(engineDatabaseUrl, { + max: enginePoolMax, + idle_timeout: enginePoolIdleReapSeconds, + max_lifetime: enginePoolMaxLifetime, + connect_timeout: enginePoolConnectionTimeout, + onnotice: () => {}, +}); + // Create accounts DB with operations layer const accountsDb = createAccountsDB(accountsSql, accountsSchema); @@ -426,6 +443,15 @@ if (engineSchemas.length > 0) { info("No engine schemas to migrate"); } +// New model (Phase 4 cutover): prepare the DB for per-space schemas and migrate +// the auth + core control-plane schemas on the single postgres.js pool. These +// run alongside the legacy schemas above; the new auth/memory paths consume +// them as they come online (4B+). +await bootstrapSpaceDatabase(db); +await migrateCore(db); +await migrateAuth(db); +info("Core + auth schemas migrated"); + // ============================================================================= // Router // ============================================================================= @@ -567,6 +593,7 @@ async function shutdown() { await accountsSql.close(); await engineSql.close(); await workerEngineSql.close(); + await db.end(); } catch (error) { reportError("Error closing database connections", error as Error); } From 7f06973e50cca5851e2f14f4a886fc95d9cb253d Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 17:36:00 +0200 Subject: [PATCH 038/156] feat(auth): device-flow consent via status (pending/approved/denied) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the device-authorization consent gate, better-auth-shaped: replace the `denied` boolean on auth.device_authorization with a `status` column (pending | approved | denied), added straight to the table def (pre-prod). Split the old authorize_device into the two real steps so OAuth success no longer auto-authorizes (closes the RFC 8628 device-phishing gap): - bind_device_user: the OAuth callback resolved the user → set user_id, status stays 'pending' - approve_device: the human consented → status 'approved' - deny_device: status 'denied' - poll_device returns the stored status straight through plus the poll-only 'expired'/'slow_down'; a bound-but-unapproved device still polls 'pending'. (Unified naming: 'approved', not a separate 'authorized'.) authStore mirrors this: DeviceAuthRow.status, new DeviceStatus type, bindDeviceUser + approveDevice replacing authorizeDevice; DevicePollStatus = DeviceStatus | 'expired' | 'slow_down'. 30 auth tests (migrate + authStore) green, full check (1051) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/auth/db.integration.test.ts | 8 ++- packages/auth/db.ts | 23 +++++-- packages/auth/index.ts | 1 + packages/auth/types.ts | 16 +++-- .../migrate/idempotent/004_device_auth.sql | 69 ++++++++++++------- .../incremental/004_device_authorization.sql | 4 +- .../auth/migrate/migrate.integration.test.ts | 52 ++++++++++---- 7 files changed, 115 insertions(+), 58 deletions(-) diff --git a/packages/auth/db.integration.test.ts b/packages/auth/db.integration.test.ts index 5fbc1b4..88ecff1 100644 --- a/packages/auth/db.integration.test.ts +++ b/packages/auth/db.integration.test.ts @@ -111,11 +111,13 @@ test("device flow: create → lookup (normalized code) → poll → authorize", const byState = await db.getDeviceByOAuthState(oauthState); expect(byState?.deviceCode).toBe(deviceCode); - // pending → authorize → authorized + // pending → bind (callback) → still pending → approve (consent) → authorized expect((await db.pollDevice(deviceCode, 0)).status).toBe("pending"); - expect(await db.authorizeDevice(deviceCode, id)).toBe(true); + expect(await db.bindDeviceUser(deviceCode, id)).toBe(true); + expect((await db.pollDevice(deviceCode, 0)).status).toBe("pending"); + expect(await db.approveDevice(deviceCode)).toBe(true); const poll = await db.pollDevice(deviceCode, 0); - expect(poll.status).toBe("authorized"); + expect(poll.status).toBe("approved"); expect(poll.userId).toBe(id); }); diff --git a/packages/auth/db.ts b/packages/auth/db.ts index 509d75a..8bb81c9 100644 --- a/packages/auth/db.ts +++ b/packages/auth/db.ts @@ -16,6 +16,7 @@ import type { CreateUserOptions, DevicePollResult, DevicePollStatus, + DeviceStatus, OAuthProvider, User, ValidatedSession, @@ -65,12 +66,16 @@ export interface AuthStore { createDeviceAuth(provider: OAuthProvider): Promise; getDeviceByUserCode(userCode: string): Promise; getDeviceByOAuthState(oauthState: string): Promise; - /** Resolve the poll state machine in one call (see core poll_device). */ + /** Resolve the poll state machine in one call (see poll_device). */ pollDevice( deviceCode: string, minIntervalSecs?: number, ): Promise; - authorizeDevice(deviceCode: string, userId: string): Promise; + /** Callback bound the resolved user (status stays 'pending' until consent). */ + bindDeviceUser(deviceCode: string, userId: string): Promise; + /** Consent: approve the bound device (→ 'approved'). */ + approveDevice(deviceCode: string): Promise; + /** Consent denied, or OAuth failed (→ 'denied'). */ denyDevice(deviceCode: string): Promise; deleteDevice(deviceCode: string): Promise; deleteExpiredDevices(): Promise; @@ -87,7 +92,7 @@ export interface DeviceAuthRow { expiresAt: Date; lastPoll: Date | null; userId: string | null; - denied: boolean; + status: DeviceStatus; createdAt: Date; } @@ -121,7 +126,7 @@ function mapDevice(row: Record): DeviceAuthRow { expiresAt: row.expires_at as Date, lastPoll: (row.last_poll as Date | null) ?? null, userId: (row.user_id as string | null) ?? null, - denied: Boolean(row.denied), + status: row.status as DeviceStatus, createdAt: row.created_at as Date, }; } @@ -254,9 +259,15 @@ export function authStore(sql: Sql, schema: string = AUTH_SCHEMA): AuthStore { }; }, - async authorizeDevice(deviceCode, userId) { + async bindDeviceUser(deviceCode, userId) { const [row] = await sql` - select ${sch}.authorize_device(${deviceCode}, ${userId}) as ok`; + select ${sch}.bind_device_user(${deviceCode}, ${userId}) as ok`; + return Boolean(row?.ok); + }, + + async approveDevice(deviceCode) { + const [row] = await sql` + select ${sch}.approve_device(${deviceCode}) as ok`; return Boolean(row?.ok); }, diff --git a/packages/auth/index.ts b/packages/auth/index.ts index d0441bd..cd864fd 100644 --- a/packages/auth/index.ts +++ b/packages/auth/index.ts @@ -15,6 +15,7 @@ export type { CreateUserOptions, DevicePollResult, DevicePollStatus, + DeviceStatus, OAuthProvider, User, ValidatedSession, diff --git a/packages/auth/types.ts b/packages/auth/types.ts index 6bfadad..af81145 100644 --- a/packages/auth/types.ts +++ b/packages/auth/types.ts @@ -52,15 +52,17 @@ export interface CreatedDeviceAuth { expiresIn: number; } -export type DevicePollStatus = - | "expired" - | "slow_down" - | "denied" - | "pending" - | "authorized"; +/** The stored device_authorization state (better-auth-shaped). */ +export type DeviceStatus = "pending" | "approved" | "denied"; + +/** + * The poll-result vocabulary returned by poll_device: the stored DeviceStatus + * (pending|approved|denied) passed straight through, plus two poll-only outcomes. + */ +export type DevicePollStatus = DeviceStatus | "expired" | "slow_down"; export interface DevicePollResult { status: DevicePollStatus; - /** Set only when status === "authorized". */ + /** Set only when status === "approved". */ userId: string | null; } diff --git a/packages/database/auth/migrate/idempotent/004_device_auth.sql b/packages/database/auth/migrate/idempotent/004_device_auth.sql index 28a1d12..1f17b84 100644 --- a/packages/database/auth/migrate/idempotent/004_device_auth.sql +++ b/packages/database/auth/migrate/idempotent/004_device_auth.sql @@ -1,5 +1,5 @@ ------------------------------------------------------------------------------- --- create_device_auth (OAuth 2.0 device flow) +-- create_device_auth (OAuth 2.0 device flow). status defaults to 'pending'. ------------------------------------------------------------------------------- create or replace function {{schema}}.create_device_auth ( _device_code text @@ -18,7 +18,6 @@ $func$ language sql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; --- shared return shape for the device lookups (unexpired rows only) ------------------------------------------------------------------------------- -- get_device_by_user_code (browser code entry; caller normalizes the code) ------------------------------------------------------------------------------- @@ -31,12 +30,12 @@ returns table , expires_at timestamptz , last_poll timestamptz , user_id uuid -, denied bool +, status text , created_at timestamptz ) as $func$ select d.device_code, d.user_code, d.provider, d.oauth_state, d.expires_at, - d.last_poll, d.user_id, d.denied, d.created_at + d.last_poll, d.user_id, d.status, d.created_at from {{schema}}.device_authorization d where d.user_code = _user_code and d.expires_at > now() $func$ language sql stable security invoker @@ -44,7 +43,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- get_device_by_oauth_state (OAuth callback) +-- get_device_by_oauth_state (OAuth callback + consent) ------------------------------------------------------------------------------- create or replace function {{schema}}.get_device_by_oauth_state(_oauth_state text) returns table @@ -55,12 +54,12 @@ returns table , expires_at timestamptz , last_poll timestamptz , user_id uuid -, denied bool +, status text , created_at timestamptz ) as $func$ select d.device_code, d.user_code, d.provider, d.oauth_state, d.expires_at, - d.last_poll, d.user_id, d.denied, d.created_at + d.last_poll, d.user_id, d.status, d.created_at from {{schema}}.device_authorization d where d.oauth_state = _oauth_state and d.expires_at > now() $func$ language sql stable security invoker @@ -68,9 +67,10 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- authorize_device (callback success → bind the user) +-- bind_device_user (callback resolved the user; status stays 'pending' until +-- the human consents). Returns false if already bound / not pending / expired. ------------------------------------------------------------------------------- -create or replace function {{schema}}.authorize_device(_device_code text, _user_id uuid) +create or replace function {{schema}}.bind_device_user(_device_code text, _user_id uuid) returns bool as $func$ with u as @@ -79,8 +79,8 @@ as $func$ set user_id = _user_id where device_code = _device_code and expires_at > now() + and status = 'pending' and user_id is null - and denied = false returning 1 ) select exists (select 1 from u) @@ -89,7 +89,28 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- deny_device +-- approve_device (the human consented). Requires a bound user + pending status. +------------------------------------------------------------------------------- +create or replace function {{schema}}.approve_device(_device_code text) +returns bool +as $func$ + with u as + ( + update {{schema}}.device_authorization + set status = 'approved' + where device_code = _device_code + and expires_at > now() + and status = 'pending' + and user_id is not null + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- deny_device (the human denied, or the OAuth step failed) ------------------------------------------------------------------------------- create or replace function {{schema}}.deny_device(_device_code text) returns bool @@ -97,10 +118,10 @@ as $func$ with u as ( update {{schema}}.device_authorization - set denied = true + set status = 'denied' where device_code = _device_code and expires_at > now() - and user_id is null + and status = 'pending' returning 1 ) select exists (select 1 from u) @@ -109,14 +130,14 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- poll_device (CLI polling — resolves the device-flow state in one call) --- Returns a status, plus the bound user when authorized: +-- poll_device (CLI polling — resolves the poll state in one call). Returns the +-- stored status (pending|approved|denied) straight through, plus two poll-only +-- outcomes: -- 'expired' — no unexpired device with this code -- 'slow_down' — polled within _min_interval_secs (last_poll NOT advanced) --- 'denied' — the user denied the request --- 'pending' — not yet authorized --- 'authorized' — bound to user_id (caller then mints a session + deletes it) --- Subsumes the old get_device_by_device_code + update_device_last_poll. +-- 'pending' — created/bound but not yet approved +-- 'denied' — the request was denied +-- 'approved' — bound user_id; the caller mints a session + deletes the device ------------------------------------------------------------------------------- create or replace function {{schema}}.poll_device ( _device_code text @@ -147,13 +168,9 @@ begin set last_poll = now() where device_code = _device_code; - if _d.denied then - return query select 'denied'::text, null::uuid; - elsif _d.user_id is not null then - return query select 'authorized'::text, _d.user_id; - else - return query select 'pending'::text, null::uuid; - end if; + -- stored status passes straight through; user_id only when approved + return query + select _d.status, case when _d.status = 'approved' then _d.user_id else null::uuid end; end; $func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp diff --git a/packages/database/auth/migrate/incremental/004_device_authorization.sql b/packages/database/auth/migrate/incremental/004_device_authorization.sql index 2754563..7953408 100644 --- a/packages/database/auth/migrate/incremental/004_device_authorization.sql +++ b/packages/database/auth/migrate/incremental/004_device_authorization.sql @@ -11,8 +11,8 @@ create table {{schema}}.device_authorization , oauth_state text not null unique -- CSRF binding for the OAuth callback , expires_at timestamptz not null -- short TTL (~15 min) , last_poll timestamptz -- rate-limiting the CLI poll -, user_id uuid references {{schema}}.users (id) on delete cascade -- null until authorized -, denied boolean not null default false +, user_id uuid references {{schema}}.users (id) on delete cascade -- bound once the callback resolves the user +, status text not null default 'pending' check (status in ('pending', 'approved', 'denied')) -- approved only after explicit consent , created_at timestamptz not null default now() ); diff --git a/packages/database/auth/migrate/migrate.integration.test.ts b/packages/database/auth/migrate/migrate.integration.test.ts index 1e635ac..134a92d 100644 --- a/packages/database/auth/migrate/migrate.integration.test.ts +++ b/packages/database/auth/migrate/migrate.integration.test.ts @@ -53,7 +53,8 @@ const EXPECTED_FUNCTIONS = [ "upsert_account", "get_account_by_provider", "create_device_auth", - "authorize_device", + "bind_device_user", + "approve_device", "poll_device", ]; @@ -328,7 +329,7 @@ describe("auth functions", () => { }); }); - test("device flow: create → lookups → poll_device → authorize", async () => { + test("device flow: create → bind → consent → authorized (and deny path)", async () => { await withTestAuth(sql, {}, async (auth) => { const s = auth.schema; const [u] = await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ @@ -350,18 +351,18 @@ describe("auth functions", () => { [oauthState], ); expect(byState[0]?.device_code).toBe(deviceCode); + expect(byState[0]?.status).toBe("pending"); const byUserCode = await sql.unsafe( `select * from ${s}.get_device_by_user_code($1)`, [userCode], ); expect(byUserCode[0]?.device_code).toBe(deviceCode); - // poll before authorization → pending (interval 0 bypasses rate limit) + // before binding → pending (interval 0 bypasses rate limit) const [p1] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ deviceCode, ]); expect(p1?.status).toBe("pending"); - expect(p1?.user_id).toBeNull(); // immediate re-poll with the default interval → slow_down const [sd] = await sql.unsafe(`select * from ${s}.poll_device($1)`, [ @@ -369,25 +370,48 @@ describe("auth functions", () => { ]); expect(sd?.status).toBe("slow_down"); - // authorize binds the user; a second authorize is a no-op - const [a] = await sql.unsafe( - `select ${s}.authorize_device($1, $2) as ok`, + // callback binds the user, but status stays pending until consent + const [b] = await sql.unsafe( + `select ${s}.bind_device_user($1, $2) as ok`, [deviceCode, userId], ); - expect(a?.ok).toBe(true); - const [a2] = await sql.unsafe( - `select ${s}.authorize_device($1, $2) as ok`, - [deviceCode, userId], + expect(b?.ok).toBe(true); + const [pBound] = await sql.unsafe( + `select * from ${s}.poll_device($1, 0)`, + [deviceCode], ); - expect(a2?.ok).toBe(false); + expect(pBound?.status).toBe("pending"); // bound but NOT yet approved + + // consent: approve → authorized; a second approve is a no-op + const [ap] = await sql.unsafe(`select ${s}.approve_device($1) as ok`, [ + deviceCode, + ]); + expect(ap?.ok).toBe(true); + const [ap2] = await sql.unsafe(`select ${s}.approve_device($1) as ok`, [ + deviceCode, + ]); + expect(ap2?.ok).toBe(false); - // poll now resolves to authorized + the bound user const [p2] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ deviceCode, ]); - expect(p2?.status).toBe("authorized"); + expect(p2?.status).toBe("approved"); expect(p2?.user_id).toBe(userId); + // deny path on a separate device → poll resolves to denied + const dc2 = crypto.randomUUID(); + await sql.unsafe( + `select ${s}.create_device_auth($1, $2, 'google', $3, now() + interval '15 min')`, + [dc2, "WXYZ-3456", crypto.randomUUID()], + ); + expect( + (await sql.unsafe(`select ${s}.deny_device($1) as ok`, [dc2]))[0]?.ok, + ).toBe(true); + const [pd] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + dc2, + ]); + expect(pd?.status).toBe("denied"); + // unknown / expired device code → expired const [ex] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ crypto.randomUUID(), From 7936fd4f15d9bc8eb571425549980740f4835e1c Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 18:03:39 +0200 Subject: [PATCH 039/156] feat(server): cut OAuth login over to the auth schema + authStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route the device flow, session validation, and me/identity RPC through the new auth-schema authStore (postgres.js pool) instead of the legacy AccountsDB. - providers: surface emailVerified (Google verified_email, GitHub primary email's verified flag); drop offline/consent prompts (login-only, no tokens) - handlers/auth: resolve user via getAccountByProvider → getUserByEmail+ upsertAccount → provisionUser; reject unverified emails; bind device then render a consent page; add POST /auth/device/approve (approve/deny); token poll only mints a session on stored status 'approved' - middleware: authenticateAccounts takes an AuthStore; Identity slimmed to {id,email,name}; me.get/identity.getByEmail/session.revoke use the authStore - inject the authStore (+ db/authSchema/coreSchema) through ServerContext, built once in index.ts - cron: sweep expired device authorizations and sessions via the authStore - delete obsolete auth/device-flow.ts; org/engine RPC stay on AccountsDB (coexistence until Phase 5) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/server/auth/device-flow.test.ts | 369 ----------- packages/server/auth/device-flow.ts | 182 ------ packages/server/auth/index.ts | 1 - packages/server/auth/providers/github.ts | 36 +- packages/server/auth/providers/google.ts | 5 +- packages/server/auth/types.ts | 2 + packages/server/context.ts | 12 +- packages/server/handlers/auth.ts | 594 ++++++++---------- packages/server/index.ts | 30 +- .../server/middleware/authenticate.test.ts | 37 +- packages/server/middleware/authenticate.ts | 22 +- packages/server/router.test.ts | 8 + packages/server/router.ts | 36 +- .../rpc/accounts/engine.integration.test.ts | 3 + packages/server/rpc/accounts/me.test.ts | 64 +- packages/server/rpc/accounts/me.ts | 58 +- .../rpc/accounts/org.integration.test.ts | 3 + packages/server/rpc/accounts/session.ts | 4 +- packages/server/rpc/accounts/types.ts | 10 +- packages/server/server.integration.test.ts | 32 +- packages/server/wiring.test.ts | 39 +- 21 files changed, 508 insertions(+), 1039 deletions(-) delete mode 100644 packages/server/auth/device-flow.test.ts delete mode 100644 packages/server/auth/device-flow.ts diff --git a/packages/server/auth/device-flow.test.ts b/packages/server/auth/device-flow.test.ts deleted file mode 100644 index 05be81e..0000000 --- a/packages/server/auth/device-flow.test.ts +++ /dev/null @@ -1,369 +0,0 @@ -/** - * Tests for OAuth device flow state management. - * - * Uses an in-memory mock of the device auth database operations. - */ -import { beforeEach, describe, expect, test } from "bun:test"; -import type { - AccountsDB, - CreateDeviceAuthParams, - DeviceAuthorization, -} from "@memory.build/accounts"; -import { - authorizeDevice, - checkPollRateLimit, - cleanupDeviceState, - cleanupExpiredStates, - createDeviceAuthorization, - denyDevice, - getDeviceStateByDeviceCode, - getDeviceStateByOAuthState, - getDeviceStateByUserCode, -} from "./device-flow"; - -/** - * Create a mock AccountsDB with in-memory device auth storage. - * Only implements the device auth methods needed for testing. - */ -function createMockDb(): AccountsDB { - const store = new Map(); - const userCodeIndex = new Map(); - const oauthStateIndex = new Map(); - - return { - // Device auth operations - create: async (params: CreateDeviceAuthParams) => { - const auth: DeviceAuthorization = { - deviceCode: params.deviceCode, - userCode: params.userCode, - provider: params.provider, - oauthState: params.oauthState, - expiresAt: params.expiresAt, - lastPoll: null, - identityId: null, - denied: false, - createdAt: new Date(), - }; - store.set(params.deviceCode, auth); - userCodeIndex.set(params.userCode, params.deviceCode); - oauthStateIndex.set(params.oauthState, params.deviceCode); - return auth; - }, - - getByDeviceCode: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - return auth; - }, - - getByUserCode: async (userCode: string) => { - // Normalize: uppercase, remove hyphen, reconstruct XXXX-XXXX - const normalized = userCode.toUpperCase().replace(/-/g, ""); - const formatted = `${normalized.slice(0, 4)}-${normalized.slice(4)}`; - const deviceCode = userCodeIndex.get(formatted); - if (!deviceCode) return null; - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - return auth; - }, - - getByOAuthState: async (oauthState: string) => { - const deviceCode = oauthStateIndex.get(oauthState); - if (!deviceCode) return null; - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - return auth; - }, - - updateLastPoll: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - const previousPoll = auth.lastPoll; - auth.lastPoll = new Date(); - if (!previousPoll) { - return null; - } - return Date.now() - previousPoll.getTime(); - }, - - authorize: async (deviceCode: string, identityId: string) => { - const auth = store.get(deviceCode); - if ( - !auth || - new Date() > auth.expiresAt || - auth.identityId !== null || - auth.denied - ) { - return false; - } - auth.identityId = identityId; - return true; - }, - - deny: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt || auth.identityId !== null) { - return false; - } - auth.denied = true; - return true; - }, - - delete: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth) return false; - userCodeIndex.delete(auth.userCode); - oauthStateIndex.delete(auth.oauthState); - store.delete(deviceCode); - return true; - }, - - deleteExpired: async () => { - const now = new Date(); - let count = 0; - for (const [deviceCode, auth] of store) { - if (now > auth.expiresAt) { - userCodeIndex.delete(auth.userCode); - oauthStateIndex.delete(auth.oauthState); - store.delete(deviceCode); - count++; - } - } - return count; - }, - - // Expose store for testing (to manually expire entries) - _store: store, - } as unknown as AccountsDB; -} - -describe("device-flow", () => { - let db: AccountsDB; - - beforeEach(() => { - db = createMockDb(); - }); - - describe("createDeviceAuthorization", () => { - test("creates authorization with required fields", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - expect(auth.deviceCode).toBeDefined(); - expect(auth.deviceCode.length).toBeGreaterThan(20); - - expect(auth.userCode).toBeDefined(); - expect(auth.userCode).toMatch(/^[A-Z0-9]{4}-[A-Z0-9]{4}$/); - - expect(auth.oauthState).toBeDefined(); - expect(auth.oauthState.length).toBeGreaterThan(10); - - expect(auth.expiresIn).toBe(900); // 15 minutes - expect(auth.interval).toBe(5); - }); - - test("creates unique codes", async () => { - const auth1 = await createDeviceAuthorization(db, "google"); - const auth2 = await createDeviceAuthorization(db, "google"); - - expect(auth1.deviceCode).not.toBe(auth2.deviceCode); - expect(auth1.userCode).not.toBe(auth2.userCode); - expect(auth1.oauthState).not.toBe(auth2.oauthState); - }); - }); - - describe("getDeviceStateByUserCode", () => { - test("finds state by user code", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const state = await getDeviceStateByUserCode(db, auth.userCode); - - expect(state).not.toBeNull(); - expect(state?.userCode).toBe(auth.userCode); - expect(state?.provider).toBe("google"); - }); - - test("normalizes user code (case insensitive, with/without hyphen)", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - // Original format - expect(await getDeviceStateByUserCode(db, auth.userCode)).not.toBeNull(); - - // Lowercase - expect( - await getDeviceStateByUserCode(db, auth.userCode.toLowerCase()), - ).not.toBeNull(); - - // Without hyphen - expect( - await getDeviceStateByUserCode(db, auth.userCode.replace("-", "")), - ).not.toBeNull(); - - // Lowercase without hyphen - expect( - await getDeviceStateByUserCode( - db, - auth.userCode.toLowerCase().replace("-", ""), - ), - ).not.toBeNull(); - }); - - test("returns null for unknown code", async () => { - expect(await getDeviceStateByUserCode(db, "XXXX-XXXX")).toBeNull(); - }); - }); - - describe("getDeviceStateByOAuthState", () => { - test("finds state by OAuth state", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const state = await getDeviceStateByOAuthState(db, auth.oauthState); - - expect(state).not.toBeNull(); - expect(state?.oauthState).toBe(auth.oauthState); - }); - - test("returns null for unknown state", async () => { - expect(await getDeviceStateByOAuthState(db, "unknown-state")).toBeNull(); - }); - }); - - describe("getDeviceStateByDeviceCode", () => { - test("finds state by device code", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const state = await getDeviceStateByDeviceCode(db, auth.deviceCode); - - expect(state).not.toBeNull(); - expect(state?.deviceCode).toBe(auth.deviceCode); - }); - - test("returns null for unknown code", async () => { - expect(await getDeviceStateByDeviceCode(db, "unknown-code")).toBeNull(); - }); - }); - - describe("checkPollRateLimit", () => { - test("returns false on first poll", async () => { - const auth = await createDeviceAuthorization(db, "google"); - expect(await checkPollRateLimit(db, auth.deviceCode)).toBe(false); - }); - - test("returns true when polling too fast", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - // First poll - await checkPollRateLimit(db, auth.deviceCode); - - // Immediate second poll should be rate limited - expect(await checkPollRateLimit(db, auth.deviceCode)).toBe(true); - }); - - test("returns false for unknown device code", async () => { - expect(await checkPollRateLimit(db, "unknown-code")).toBe(false); - }); - }); - - describe("authorizeDevice", () => { - test("marks device as authorized", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const identityId = "019d694f-79f6-7595-8faf-b70b01c11f98"; - - const result = await authorizeDevice(db, auth.deviceCode, identityId); - expect(result).toBe(true); - - const state = await getDeviceStateByDeviceCode(db, auth.deviceCode); - expect(state?.identityId).toBe(identityId); - }); - - test("returns false for unknown device code", async () => { - const result = await authorizeDevice( - db, - "unknown-code", - "019d694f-79f6-7595-8faf-b70b01c11f98", - ); - expect(result).toBe(false); - }); - }); - - describe("denyDevice", () => { - test("marks device as denied", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - const result = await denyDevice(db, auth.deviceCode); - expect(result).toBe(true); - - const state = await getDeviceStateByDeviceCode(db, auth.deviceCode); - expect(state?.denied).toBe(true); - }); - - test("returns false for unknown device code", async () => { - const result = await denyDevice(db, "unknown-code"); - expect(result).toBe(false); - }); - }); - - describe("cleanupDeviceState", () => { - test("removes device state", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - await cleanupDeviceState(db, auth.deviceCode); - - expect(await getDeviceStateByDeviceCode(db, auth.deviceCode)).toBeNull(); - expect(await getDeviceStateByUserCode(db, auth.userCode)).toBeNull(); - expect(await getDeviceStateByOAuthState(db, auth.oauthState)).toBeNull(); - }); - - test("handles unknown device code gracefully", async () => { - // Should not throw - await cleanupDeviceState(db, "unknown-code"); - }); - }); - - describe("cleanupExpiredStates", () => { - test("removes expired states", async () => { - // Create a state - const auth = await createDeviceAuthorization(db, "google"); - - // Manually expire it by modifying the store - const store = ( - db as unknown as { _store: Map } - )._store; - const state = store.get(auth.deviceCode); - if (state) { - state.expiresAt = new Date(Date.now() - 1000); // Expired 1 second ago - } - - // Cleanup - const cleaned = await cleanupExpiredStates(db); - expect(cleaned).toBeGreaterThanOrEqual(1); - - // State should be gone - expect(await getDeviceStateByDeviceCode(db, auth.deviceCode)).toBeNull(); - }); - }); - - describe("state expiration", () => { - test("expired state returns null on lookup", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - // Manually expire it - const store = ( - db as unknown as { _store: Map } - )._store; - const state = store.get(auth.deviceCode); - if (state) { - state.expiresAt = new Date(Date.now() - 1000); - } - - // Lookup should return null - expect(await getDeviceStateByDeviceCode(db, auth.deviceCode)).toBeNull(); - expect(await getDeviceStateByUserCode(db, auth.userCode)).toBeNull(); - expect(await getDeviceStateByOAuthState(db, auth.oauthState)).toBeNull(); - }); - }); -}); diff --git a/packages/server/auth/device-flow.ts b/packages/server/auth/device-flow.ts deleted file mode 100644 index ea1ee4a..0000000 --- a/packages/server/auth/device-flow.ts +++ /dev/null @@ -1,182 +0,0 @@ -/** - * OAuth device flow state management. - * - * Manages device authorization state in PostgreSQL for multi-node support. - * State is persisted to database and cleaned up via cron. - */ - -import type { AccountsDB, DeviceAuthorization } from "@memory.build/accounts"; -import type { OAuthProvider } from "./types"; - -/** Device code expiration (15 minutes) */ -const DEVICE_CODE_EXPIRY_MS = 15 * 60 * 1000; - -/** Minimum polling interval (5 seconds) */ -const MIN_POLL_INTERVAL_MS = 5 * 1000; - -/** User code length (8 characters, alphanumeric, easy to type) */ -const USER_CODE_LENGTH = 8; - -/** Device code length (32 bytes, URL-safe base64) */ -const DEVICE_CODE_LENGTH = 32; - -/** OAuth state length (16 bytes, URL-safe base64) */ -const OAUTH_STATE_LENGTH = 16; - -/** - * Generate a cryptographically secure random string. - */ -function generateRandomString(length: number): string { - const bytes = new Uint8Array(length); - crypto.getRandomValues(bytes); - return Buffer.from(bytes).toString("base64url"); -} - -/** - * Generate a user-friendly code (uppercase alphanumeric, no ambiguous chars). - * Format: XXXX-XXXX (8 chars with hyphen separator for readability) - */ -function generateUserCode(): string { - // Exclude ambiguous characters: 0, O, 1, I, L - const chars = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"; - let code = ""; - const bytes = new Uint8Array(USER_CODE_LENGTH); - crypto.getRandomValues(bytes); - for (const byte of bytes) { - code += chars[byte % chars.length]; - } - // Insert hyphen for readability: XXXX-XXXX - return `${code.slice(0, 4)}-${code.slice(4)}`; -} - -/** - * Create a new device authorization. - * - * @returns Device code, user code, and expiry info - */ -export async function createDeviceAuthorization( - db: AccountsDB, - provider: OAuthProvider, -): Promise<{ - deviceCode: string; - userCode: string; - oauthState: string; - expiresIn: number; - interval: number; -}> { - const deviceCode = generateRandomString(DEVICE_CODE_LENGTH); - const userCode = generateUserCode(); - const oauthState = generateRandomString(OAUTH_STATE_LENGTH); - const expiresAt = new Date(Date.now() + DEVICE_CODE_EXPIRY_MS); - - await db.create({ - deviceCode, - userCode, - provider, - oauthState, - expiresAt, - }); - - return { - deviceCode, - userCode, - oauthState, - expiresIn: Math.floor(DEVICE_CODE_EXPIRY_MS / 1000), - interval: Math.floor(MIN_POLL_INTERVAL_MS / 1000), - }; -} - -/** - * Get device state by user code. - * Used when user enters code in browser. - */ -export async function getDeviceStateByUserCode( - db: AccountsDB, - userCode: string, -): Promise { - return db.getByUserCode(userCode); -} - -/** - * Get device state by OAuth state parameter. - * Used in OAuth callback to find the device being authorized. - */ -export async function getDeviceStateByOAuthState( - db: AccountsDB, - oauthState: string, -): Promise { - return db.getByOAuthState(oauthState); -} - -/** - * Get device state by device code. - * Used for polling. - */ -export async function getDeviceStateByDeviceCode( - db: AccountsDB, - deviceCode: string, -): Promise { - return db.getByDeviceCode(deviceCode); -} - -/** - * Check if polling is too fast (rate limiting). - * Also updates the last poll timestamp. - * - * @returns true if client should slow down - */ -export async function checkPollRateLimit( - db: AccountsDB, - deviceCode: string, -): Promise { - const elapsedMs = await db.updateLastPoll(deviceCode); - - // First poll or not found - if (elapsedMs === null) { - return false; - } - - // Too fast if less than minimum interval - return elapsedMs < MIN_POLL_INTERVAL_MS; -} - -/** - * Mark device as authorized with an identity. - * Called after successful OAuth callback. - */ -export async function authorizeDevice( - db: AccountsDB, - deviceCode: string, - identityId: string, -): Promise { - return db.authorize(deviceCode, identityId); -} - -/** - * Mark device as denied. - * Called if user denies access. - */ -export async function denyDevice( - db: AccountsDB, - deviceCode: string, -): Promise { - return db.deny(deviceCode); -} - -/** - * Clean up device state after completion or expiry. - */ -export async function cleanupDeviceState( - db: AccountsDB, - deviceCode: string, -): Promise { - await db.delete(deviceCode); -} - -/** - * Clean up all expired device states. - * Called by cron job. - */ -export async function cleanupExpiredStates(db: AccountsDB): Promise { - return db.deleteExpired(); -} diff --git a/packages/server/auth/index.ts b/packages/server/auth/index.ts index c7079e6..516b778 100644 --- a/packages/server/auth/index.ts +++ b/packages/server/auth/index.ts @@ -2,6 +2,5 @@ * OAuth authentication module exports. */ -export * from "./device-flow"; export * from "./providers"; export * from "./types"; diff --git a/packages/server/auth/providers/github.ts b/packages/server/auth/providers/github.ts index b4578ac..a790506 100644 --- a/packages/server/auth/providers/github.ts +++ b/packages/server/auth/providers/github.ts @@ -145,36 +145,32 @@ export async function fetchGitHubUserInfo( id: number; login: string; name: string | null; - email: string | null; }; - // If email is not public, fetch from emails endpoint - let email = userData.email; - if (!email) { - email = await fetchGitHubPrimaryEmail(accessToken); - } - - if (!email) { + // Always resolve the email via /user/emails so we get its `verified` flag + // (the public profile email field carries no verification signal). + const primary = await fetchGitHubPrimaryEmail(accessToken); + if (!primary) { throw new Error( - "GitHub account does not have a verified email address. Please add and verify an email in your GitHub settings.", + "GitHub account does not have an email address. Please add one in your GitHub settings.", ); } return { providerAccountId: String(userData.id), - email, + email: primary.email, + emailVerified: primary.verified, name: userData.name || userData.login, }; } /** - * Fetch user's primary verified email from GitHub. - * - * @param accessToken - OAuth access token + * Fetch the user's primary email from GitHub, with its verified flag. + * (Returns the primary email if present, else the first; null if none.) */ async function fetchGitHubPrimaryEmail( accessToken: string, -): Promise { +): Promise<{ email: string; verified: boolean } | null> { const response = await fetch("https://api.github.com/user/emails", { headers: { Authorization: `Bearer ${accessToken}`, @@ -184,7 +180,6 @@ async function fetchGitHubPrimaryEmail( }); if (!response.ok) { - // If we can't fetch emails, return null and let caller handle it return null; } @@ -194,13 +189,6 @@ async function fetchGitHubPrimaryEmail( verified: boolean; }>; - // Find primary verified email - const primary = emails.find((e) => e.primary && e.verified); - if (primary) { - return primary.email; - } - - // Fall back to any verified email - const verified = emails.find((e) => e.verified); - return verified?.email ?? null; + const chosen = emails.find((e) => e.primary) ?? emails[0]; + return chosen ? { email: chosen.email, verified: chosen.verified } : null; } diff --git a/packages/server/auth/providers/google.ts b/packages/server/auth/providers/google.ts index 1306cd1..c130cef 100644 --- a/packages/server/auth/providers/google.ts +++ b/packages/server/auth/providers/google.ts @@ -40,14 +40,14 @@ export function getGoogleConfig(): OAuthProviderConfig { */ export function buildGoogleAuthUrl(state: string, redirectUri: string): string { const config = getGoogleConfig(); + // Login-only: we use the access token once (to read the profile) and never + // store it, so we don't request offline access or force a consent prompt. const params = new URLSearchParams({ client_id: config.clientId, redirect_uri: redirectUri, response_type: "code", scope: config.scopes.join(" "), state, - access_type: "offline", - prompt: "consent", }); return `${config.authorizationUrl}?${params.toString()}`; @@ -137,6 +137,7 @@ export async function fetchGoogleUserInfo( return { providerAccountId: data.id, email: data.email, + emailVerified: Boolean(data.verified_email), name: data.name || data.email.split("@")[0] || "User", }; } diff --git a/packages/server/auth/types.ts b/packages/server/auth/types.ts index b67b189..2b8bb3f 100644 --- a/packages/server/auth/types.ts +++ b/packages/server/auth/types.ts @@ -114,6 +114,8 @@ export interface OAuthUserInfo { providerAccountId: string; /** User's email */ email: string; + /** Whether the provider has verified the user controls this email. */ + emailVerified: boolean; /** User's display name */ name: string; } diff --git a/packages/server/context.ts b/packages/server/context.ts index e2ceb3f..2c37878 100644 --- a/packages/server/context.ts +++ b/packages/server/context.ts @@ -1,18 +1,28 @@ import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { SQL } from "bun"; +import type { Sql } from "postgres"; /** * Server-wide context containing database connections. * Passed to createRouter() at startup. */ export interface ServerContext { - /** Accounts database operations */ + /** Accounts database operations (legacy: org/engine, until Phase 5) */ accountsDb: AccountsDB; /** Accounts database pool (for health checks) */ accountsSql: SQL; /** Engine database pool (EngineDB created per-request based on slug) */ engineSql: SQL; + /** New-model pool (postgres.js): auth + core + per-space schemas, one DB */ + db: Sql; + /** Auth store (auth schema): me/session/identity/device + OAuth accounts */ + auth: AuthStore; + /** The auth schema name */ + authSchema: string; + /** The core control-plane schema name */ + coreSchema: string; /** Embedding config for semantic search */ embeddingConfig: EmbeddingConfig; /** Base URL for API callbacks (e.g., "https://memory.build") */ diff --git a/packages/server/handlers/auth.ts b/packages/server/handlers/auth.ts index 7a14c74..7f900a7 100644 --- a/packages/server/handlers/auth.ts +++ b/packages/server/handlers/auth.ts @@ -1,55 +1,44 @@ /** - * OAuth device flow HTTP handlers. + * OAuth device flow HTTP handlers (new model: authStore + provisionUser). * * Endpoints: - * - POST /api/v1/auth/device/code - CLI initiates device flow - * - POST /api/v1/auth/device/token - CLI polls for token - * - GET /api/v1/auth/device/verify - User enters code (HTML form) - * - POST /api/v1/auth/device/verify - User submits code - * - GET /api/v1/auth/callback/:provider - OAuth callback + * - POST /api/v1/auth/device/code - CLI initiates device flow + * - POST /api/v1/auth/device/token - CLI polls for token + * - GET /api/v1/auth/device/verify - User enters code (HTML form) + * - POST /api/v1/auth/device/verify - User submits code -> OAuth redirect + * - GET /api/v1/auth/callback/:provider - OAuth callback -> consent page + * - POST /api/v1/auth/device/approve - User approves/denies (consent) */ -import type { AccountsDB, Identity } from "@memory.build/accounts"; -import { type EngineConfig, provisionEngine } from "@memory.build/engine"; -import { setLocalEngineTimeouts } from "@memory.build/engine/ops/_tx"; +import type { AuthStore, OAuthProvider } from "@memory.build/auth"; import { info, reportError } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import { - authorizeDevice, - checkPollRateLimit, - cleanupDeviceState, - createDeviceAuthorization, - denyDevice, - getDeviceStateByDeviceCode, - getDeviceStateByOAuthState, - getDeviceStateByUserCode, -} from "../auth/device-flow"; +import type { Sql } from "postgres"; import { buildAuthUrl, exchangeCode, fetchUserInfo } from "../auth/providers"; -import type { OAuthProvider } from "../auth/types"; -import { embeddingConstants } from "../config"; +import { provisionUser } from "../provision"; import type { RouteParams } from "../router"; import { error, html, json } from "../util/response"; +/** Min CLI poll interval (seconds) — matches poll_device's default. */ +const POLL_INTERVAL_SECONDS = 5; + /** - * Context needed for auth handlers. + * Context for the auth handlers. `auth` is bound to the auth schema; `db` + + * schema names are for provisionUser's atomic cross-schema transaction. */ export interface AuthHandlerContext { - /** AccountsDB instance */ - db: AccountsDB; - /** Base URL for callbacks (e.g., "https://memory.build") */ + auth: AuthStore; + db: Sql; + authSchema: string; + coreSchema: string; baseUrl: string; - /** Engine database pool (for provisioning default engine on signup) */ - engineSql: SQL; - /** Application version for migration tracking */ - serverVersion: string; +} + +function isProvider(p: string | undefined): p is OAuthProvider { + return p === "google" || p === "github"; } /** - * POST /api/v1/auth/device/code - * - * CLI initiates device flow. - * Request: { provider: "google" } - * Response: { deviceCode, userCode, verificationUri, expiresIn, interval } + * POST /api/v1/auth/device/code — CLI initiates the device flow. */ export async function deviceCodeHandler( request: Request, @@ -58,16 +47,13 @@ export async function deviceCodeHandler( if (request.method !== "POST") { return error("Method not allowed", 405, "METHOD_NOT_ALLOWED"); } - let body: { provider?: string }; try { body = (await request.json()) as { provider?: string }; } catch { return error("Invalid JSON body", 400, "INVALID_REQUEST"); } - - const provider = body.provider; - if (provider !== "google" && provider !== "github") { + if (!isProvider(body.provider)) { return error( "Invalid provider. Must be 'google' or 'github'", 400, @@ -75,24 +61,18 @@ export async function deviceCodeHandler( ); } - const auth = await createDeviceAuthorization(ctx.db, provider); - + const device = await ctx.auth.createDeviceAuth(body.provider); return json({ - deviceCode: auth.deviceCode, - userCode: auth.userCode, + deviceCode: device.deviceCode, + userCode: device.userCode, verificationUri: `${ctx.baseUrl}/api/v1/auth/device/verify`, - expiresIn: auth.expiresIn, - interval: auth.interval, + expiresIn: device.expiresIn, + interval: POLL_INTERVAL_SECONDS, }); } /** - * POST /api/v1/auth/device/token - * - * CLI polls for token. - * Request: { deviceCode: "..." } - * Response (pending): { error: "authorization_pending" } - * Response (success): { sessionToken, identity: { id, email, name } } + * POST /api/v1/auth/device/token — CLI polls for the session token. */ export async function deviceTokenHandler( request: Request, @@ -101,156 +81,82 @@ export async function deviceTokenHandler( if (request.method !== "POST") { return error("Method not allowed", 405, "METHOD_NOT_ALLOWED"); } - let body: { deviceCode?: string }; try { body = (await request.json()) as { deviceCode?: string }; } catch { return error("Invalid JSON body", 400, "INVALID_REQUEST"); } - const deviceCode = body.deviceCode; if (!deviceCode || typeof deviceCode !== "string") { return error("Missing deviceCode", 400, "INVALID_REQUEST"); } - // Check rate limit first (also updates last_poll timestamp) - const tooFast = await checkPollRateLimit(ctx.db, deviceCode); - if (tooFast) { - return json({ error: "slow_down" }, 400); - } - - // Check if device code exists - const state = await getDeviceStateByDeviceCode(ctx.db, deviceCode); - if (!state) { - return json({ error: "expired_token" }, 400); - } - - // Check if denied - if (state.denied) { - await cleanupDeviceState(ctx.db, deviceCode); - return json({ error: "access_denied" }, 400); - } - - // Check if authorized - if (!state.identityId) { - return json({ error: "authorization_pending" }, 400); - } - - // Get identity and create session - const identity = await ctx.db.getIdentity(state.identityId); - if (!identity) { - return error("Identity not found", 500, "INTERNAL_ERROR"); + const poll = await ctx.auth.pollDevice(deviceCode); + switch (poll.status) { + case "slow_down": + return json({ error: "slow_down" }, 400); + case "expired": + return json({ error: "expired_token" }, 400); + case "denied": + await ctx.auth.deleteDevice(deviceCode); + return json({ error: "access_denied" }, 400); + case "pending": + return json({ error: "authorization_pending" }, 400); + case "approved": { + if (!poll.userId) { + return error("Approved device has no user", 500, "INTERNAL_ERROR"); + } + const user = await ctx.auth.getUser(poll.userId); + if (!user) { + return error("User not found", 500, "INTERNAL_ERROR"); + } + const session = await ctx.auth.createSession(poll.userId); + await ctx.auth.deleteDevice(deviceCode); + return json({ + sessionToken: session.token, + identity: { id: user.id, email: user.email, name: user.name }, + }); + } } - - // Create session - const sessionResult = await ctx.db.createSession({ - identityId: identity.id, - }); - - // Cleanup device state - await cleanupDeviceState(ctx.db, deviceCode); - - return json({ - sessionToken: sessionResult.rawToken, - identity: { - id: identity.id, - email: identity.email, - name: identity.name, - }, - }); } /** - * GET /api/v1/auth/device/verify - * - * User visits this page to enter their code. - * Returns an HTML form. + * GET /api/v1/auth/device/verify — the page where the user enters their code. */ -export async function deviceVerifyGetHandler( +export function deviceVerifyGetHandler( _request: Request, _ctx: AuthHandlerContext, -): Promise { +): Response { const htmlContent = ` Sign in to Memory Engine - + ${PAGE_STYLE}

Sign in to Memory Engine

Enter the code shown in your CLI

-
`; - - // The form POSTs to 'self', but the response is a 302 redirect to the - // OAuth provider. Browsers enforce form-action on the full redirect chain, - // so we must whitelist the OAuth provider origins here. + // The form POSTs to self, then we 302 to the OAuth provider; browsers enforce + // form-action across the redirect chain, so whitelist the provider origins. const csp = "default-src 'none'; style-src 'unsafe-inline'; form-action 'self' https://accounts.google.com https://github.com"; - return html(htmlContent, 200, csp); } /** - * POST /api/v1/auth/device/verify - * - * User submits code. If valid, redirect to OAuth provider. + * POST /api/v1/auth/device/verify — user submitted a code; redirect to OAuth. */ export async function deviceVerifyPostHandler( request: Request, @@ -258,229 +164,247 @@ export async function deviceVerifyPostHandler( ): Promise { const formData = await request.formData(); const userCode = formData.get("user_code"); - if (!userCode || typeof userCode !== "string") { return html(errorPage("Please enter a code"), 400); } - // Find device state - const state = await getDeviceStateByUserCode(ctx.db, userCode); - if (!state) { + const device = await ctx.auth.getDeviceByUserCode(userCode); + if (!device) { return html(errorPage("Invalid or expired code. Please try again."), 400); } - // Redirect to OAuth provider - const redirectUri = `${ctx.baseUrl}/api/v1/auth/callback/${state.provider}`; - const authUrl = buildAuthUrl(state.provider, state.oauthState, redirectUri); - - return new Response(null, { - status: 302, - headers: { Location: authUrl }, - }); + const redirectUri = `${ctx.baseUrl}/api/v1/auth/callback/${device.provider}`; + const authUrl = buildAuthUrl(device.provider, device.oauthState, redirectUri); + return new Response(null, { status: 302, headers: { Location: authUrl } }); } /** - * GET /api/v1/auth/callback/:provider + * GET /api/v1/auth/callback/:provider — OAuth callback. * - * OAuth callback. Exchange code for tokens, create/link identity. + * Resolves the user (account → verified email → provision), binds them to the + * device (status stays 'pending'), and shows the consent page. Authorization + * only happens when the human approves (POST /device/approve). */ export async function oauthCallbackHandler( request: Request, params: RouteParams, ctx: AuthHandlerContext, ): Promise { - const provider = params.provider as OAuthProvider; - if (provider !== "google" && provider !== "github") { + if (!isProvider(params.provider)) { return html(errorPage("Unknown OAuth provider"), 400); } + const provider = params.provider; const url = new URL(request.url); const code = url.searchParams.get("code"); const oauthState = url.searchParams.get("state"); const errorParam = url.searchParams.get("error"); - // Check for OAuth error if (errorParam) { const errorDesc = url.searchParams.get("error_description") || errorParam; - // If we have state, mark device as denied if (oauthState) { - const deviceState = await getDeviceStateByOAuthState(ctx.db, oauthState); - if (deviceState) { - await denyDevice(ctx.db, deviceState.deviceCode); - } + const device = await ctx.auth.getDeviceByOAuthState(oauthState); + if (device) await ctx.auth.denyDevice(device.deviceCode); } return html(errorPage(`OAuth error: ${errorDesc}`), 400); } - if (!code || !oauthState) { return html(errorPage("Missing code or state parameter"), 400); } - // Find device state by OAuth state - const deviceState = await getDeviceStateByOAuthState(ctx.db, oauthState); - if (!deviceState) { + const device = await ctx.auth.getDeviceByOAuthState(oauthState); + if (!device) { return html( - errorPage( - "Invalid or expired session. Please restart the sign-in process.", - ), + errorPage("Invalid or expired session. Please restart the sign-in."), 400, ); } const redirectUri = `${ctx.baseUrl}/api/v1/auth/callback/${provider}`; - try { - // Exchange code for tokens const tokens = await exchangeCode(provider, code, redirectUri); - - // Fetch user info const userInfo = await fetchUserInfo(provider, tokens.accessToken); - // Find or create identity - let identity = await ctx.db.getIdentityByEmail(userInfo.email); - if (!identity) { - // Create new identity and provision personal account - identity = await ctx.db.createIdentity({ - email: userInfo.email, - name: userInfo.name, - }); - await provisionPersonalAccount(ctx, identity); + // Reject unverified emails — the gate that prevents account-takeover via a + // provider asserting someone else's address. + if (!userInfo.emailVerified) { + await ctx.auth.denyDevice(device.deviceCode); + return html( + errorPage( + `Your ${provider} email (${userInfo.email}) is not verified. Verify it with ${provider} and try again.`, + ), + 400, + ); } - // Link OAuth account (upserts if exists). Login-only: we do not persist - // the provider tokens — they were used above only to fetch the identity. - await ctx.db.linkOAuthAccount({ - identityId: identity.id, + // Resolve the user: existing account → existing verified email → new user. + let userId: string; + const account = await ctx.auth.getAccountByProvider( provider, - providerAccountId: userInfo.providerAccountId, - email: userInfo.email, - }); - - // Mark device as authorized - await authorizeDevice(ctx.db, deviceState.deviceCode, identity.id); + userInfo.providerAccountId, + ); + if (account) { + userId = account.userId; + } else { + const byEmail = await ctx.auth.getUserByEmail(userInfo.email); + if (byEmail) { + // verified (gated above) → safe to link this provider to the user + userId = byEmail.id; + await ctx.auth.upsertAccount( + userId, + provider, + userInfo.providerAccountId, + ); + } else { + const result = await provisionUser( + ctx.db, + { auth: ctx.authSchema, core: ctx.coreSchema }, + { + email: userInfo.email, + name: userInfo.name, + provider, + accountId: userInfo.providerAccountId, + emailVerified: true, + }, + ); + userId = result.userId; + info("Provisioned new user", { email: userInfo.email }); + } + } - // Show success page - return html(successPage()); + // Bind the user; the device stays 'pending' until the human consents. + await ctx.auth.bindDeviceUser(device.deviceCode, userId); + return html( + consentPage(userInfo.email, device.userCode, provider, oauthState), + 200, + CONSENT_CSP, + ); } catch (err) { reportError("OAuth callback error", err as Error); return html(errorPage("Authentication failed. Please try again."), 500); } } -// ============================================================================= -// Personal Account Provisioning -// ============================================================================= - /** - * Provision a personal account for a newly created identity. - * - * Creates a personal org (with the identity as owner) and a default - * memory engine within that org. This runs during first login so the - * user has an immediate, working environment. + * POST /api/v1/auth/device/approve — the human approves (or denies) the device. + * The browser only ever carries the oauth_state, never the device_code. */ -async function provisionPersonalAccount( +export async function deviceApproveHandler( + request: Request, ctx: AuthHandlerContext, - identity: Identity, -): Promise { - const { db, engineSql, serverVersion } = ctx; - - const org = await db.withTransaction(async (txDb) => { - // Create personal org - const newOrg = await txDb.createOrg({ name: "Personal" }); - - // Add identity as owner - await txDb.addMember(newOrg.id, identity.id, "owner"); - - // Create default engine record - const engine = await txDb.createEngine({ - orgId: newOrg.id, - name: "default", - }); - - // Provision the engine schema in the engine database - const engineConfig: EngineConfig = { - embedding_dimensions: embeddingConstants.dimensions, - bm25_text_config: engine.language, - }; - - try { - await provisionEngine( - engineSql, - engine.slug, - engineConfig, - serverVersion, - engine.shardId, - ); - } catch (err) { - // Clean up partially-created schema - const schema = `me_${engine.slug}`; - try { - await engineSql.begin(async (tx) => { - await tx.unsafe(`set local pgdog.shard to ${engine.shardId}`); - await setLocalEngineTimeouts(tx); - await tx.unsafe(`drop schema if exists ${schema} cascade`); - }); - } catch { - // Log but don't mask original error - } - throw err; - } +): Promise { + const formData = await request.formData(); + const oauthState = formData.get("oauth_state"); + const decision = formData.get("decision"); + if (typeof oauthState !== "string") { + return html(errorPage("Missing state."), 400); + } - return newOrg; - }); + const device = await ctx.auth.getDeviceByOAuthState(oauthState); + if (!device) { + return html( + errorPage("Invalid or expired session. Please restart the sign-in."), + 400, + ); + } - info("Provisioned personal account", { - email: identity.email, - orgId: org.id, - }); + if (decision === "deny") { + await ctx.auth.denyDevice(device.deviceCode); + return html(deniedPage()); + } + + const ok = await ctx.auth.approveDevice(device.deviceCode); + if (!ok) { + return html( + errorPage("This request was already handled or has expired."), + 400, + ); + } + return html(successPage()); } -/** - * Generate error HTML page. - */ -function errorPage(message: string): string { - return ` - - - - - Error - Memory Engine - + input:focus { outline: none; border-color: #0066cc; } + button { + width: 100%; padding: 16px; font-size: 16px; color: white; + border: none; border-radius: 8px; cursor: pointer; margin-bottom: 12px; + } + .primary { background: #0066cc; } .primary:hover { background: #0052a3; } + .secondary { background: #888; } .secondary:hover { background: #666; } + .checkmark { font-size: 64px; margin-bottom: 16px; } + a { display: inline-block; padding: 12px 24px; background: #0066cc; color: white; text-decoration: none; border-radius: 8px; } + `; + +/** Consent page: shown after OAuth; the user explicitly approves the device. */ +function consentPage( + email: string, + userCode: string, + provider: string, + oauthState: string, +): string { + return ` + + + + + Approve sign-in - Memory Engine + ${PAGE_STYLE}
-

Error

+

Approve this sign-in?

+

A device is requesting access to your memory as + ${escapeHtml(email)} (via ${escapeHtml(provider)}).

+ Only approve if you started this and the code below matches the one your + device shows: ${escapeHtml(userCode)}

+
+ + + +
+
+ +`; +} + +const CONSENT_CSP = + "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'"; + +function errorPage(message: string): string { + return ` + + + + + Error - Memory Engine + ${PAGE_STYLE} + + +
+

Error

${escapeHtml(message)}

Try Again
@@ -488,9 +412,6 @@ function errorPage(message: string): string { `; } -/** - * Generate success HTML page. - */ function successPage(): string { return ` @@ -498,37 +419,11 @@ function successPage(): string { Success - Memory Engine - + ${PAGE_STYLE}
-
✓
+
✓

You're signed in!

You can close this window and return to your CLI.

@@ -536,9 +431,24 @@ function successPage(): string { `; } -/** - * Escape HTML entities. - */ +function deniedPage(): string { + return ` + + + + + Denied - Memory Engine + ${PAGE_STYLE} + + +
+

Request denied

+

No access was granted. You can close this window.

+
+ +`; +} + function escapeHtml(text: string): string { return text .replace(/&/g, "&") diff --git a/packages/server/index.ts b/packages/server/index.ts index e322489..2da225d 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -1,6 +1,7 @@ // packages/server/index.ts import { createAccountsDB } from "@memory.build/accounts"; import { migrate as migrateAccounts } from "@memory.build/accounts/migrate/runner"; +import { authStore } from "@memory.build/auth"; import { bootstrapSpaceDatabase, migrateAuth, @@ -163,6 +164,10 @@ const deviceFlowCleanupCron = const accountsSchema = process.env.ACCOUNTS_SCHEMA || "accounts"; +// New-model schema names (single DB, postgres.js pool): auth + core control plane. +const authSchema = process.env.AUTH_SCHEMA || "auth"; +const coreSchema = process.env.CORE_SCHEMA || "core"; + const workerCount = parseIntEnv( "WORKER_COUNT", process.env.WORKER_COUNT || "", @@ -372,6 +377,9 @@ const db = postgres(engineDatabaseUrl, { // Create accounts DB with operations layer const accountsDb = createAccountsDB(accountsSql, accountsSchema); +// Auth store (auth schema) on the new-model postgres.js pool. +const auth = authStore(db, authSchema); + // ============================================================================= // Database Bootstrap & Migrations (blocking — server won't serve until current) // ============================================================================= @@ -460,6 +468,10 @@ const serverContext: ServerContext = { accountsDb, accountsSql, engineSql, + db, + auth, + authSchema, + coreSchema, embeddingConfig, apiBaseUrl, serverVersion: SERVER_VERSION, @@ -512,16 +524,26 @@ info("Embedding worker pool started", { workers: workerCount }); // Cleanup Jobs // ============================================================================= -// Cleanup expired device authorizations on a cron schedule (UTC) +// Sweep expired device authorizations and sessions on a cron schedule (UTC). +// Both live in the auth schema now; terminal device states delete themselves on +// poll, so this only reclaims rows that were abandoned before completing. const cleanupCron = Bun.cron(deviceFlowCleanupCron, async () => { try { - const count = await accountsDb.deleteExpired(); - if (count > 0) { - info("Cleaned up expired device authorizations", { count }); + const devices = await auth.deleteExpiredDevices(); + if (devices > 0) { + info("Cleaned up expired device authorizations", { count: devices }); } } catch (error) { reportError("Failed to cleanup device authorizations", error as Error); } + try { + const sessions = await auth.cleanupExpiredSessions(); + if (sessions > 0) { + info("Cleaned up expired sessions", { count: sessions }); + } + } catch (error) { + reportError("Failed to cleanup expired sessions", error as Error); + } }); // ============================================================================= diff --git a/packages/server/middleware/authenticate.test.ts b/packages/server/middleware/authenticate.test.ts index 4492193..b485c6a 100644 --- a/packages/server/middleware/authenticate.test.ts +++ b/packages/server/middleware/authenticate.test.ts @@ -1,5 +1,6 @@ import { describe, expect, mock, test } from "bun:test"; import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EngineDB } from "@memory.build/engine"; import type { SQL } from "bun"; import { @@ -44,17 +45,24 @@ describe("authenticateAccounts", () => { id: "identity-123", email: "test@example.com", name: "Test User", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: null, + }; + + // A validate_session row: the session plus its user. + const validatedSession = { + sessionId: "session-1", + userId: mockIdentity.id, + email: mockIdentity.email, + name: mockIdentity.name, + expiresAt: new Date("2026-12-31T00:00:00Z"), }; test("returns 401 when no Authorization header", async () => { const request = new Request("http://localhost/test"); - const mockDb = { + const mockAuth = { validateSession: mock(() => Promise.resolve(null)), - } as unknown as AccountsDB; + } as unknown as AuthStore; - const result = await authenticateAccounts(request, mockDb); + const result = await authenticateAccounts(request, mockAuth); expect(result.ok).toBe(false); if (!result.ok) { @@ -67,9 +75,9 @@ describe("authenticateAccounts", () => { headers: { Authorization: "Bearer invalid-token" }, }); const validateSession = mock(() => Promise.resolve(null)); - const mockDb = { validateSession } as unknown as AccountsDB; + const mockAuth = { validateSession } as unknown as AuthStore; - const result = await authenticateAccounts(request, mockDb); + const result = await authenticateAccounts(request, mockAuth); expect(result.ok).toBe(false); if (!result.ok) { @@ -82,16 +90,11 @@ describe("authenticateAccounts", () => { const request = new Request("http://localhost/test", { headers: { Authorization: "Bearer valid-token" }, }); - const mockDb = { - validateSession: mock(() => - Promise.resolve({ - session: { id: "session-1", identityId: mockIdentity.id }, - identity: mockIdentity, - }), - ), - } as unknown as AccountsDB; - - const result = await authenticateAccounts(request, mockDb); + const mockAuth = { + validateSession: mock(() => Promise.resolve(validatedSession)), + } as unknown as AuthStore; + + const result = await authenticateAccounts(request, mockAuth); expect(result.ok).toBe(true); if (result.ok && result.context.type === "accounts") { diff --git a/packages/server/middleware/authenticate.ts b/packages/server/middleware/authenticate.ts index 0a636fb..fbe2470 100644 --- a/packages/server/middleware/authenticate.ts +++ b/packages/server/middleware/authenticate.ts @@ -1,4 +1,5 @@ import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import { createEngineDB, type EngineDB, @@ -23,14 +24,13 @@ export const ENGINE_SCHEMA_PREFIX = "me_"; // ============================================================================= /** - * Identity from accounts DB (for accounts RPC). + * The authenticated principal for accounts RPC — the session-validated user + * (auth.users), kept lightweight; me.get fetches the full record when needed. */ export interface Identity { id: string; email: string; name: string; - createdAt: Date; - updatedAt: Date | null; } /** @@ -119,11 +119,11 @@ export function extractBearerToken(request: Request): string | null { /** * Authenticate request for accounts RPC. - * Validates session token and returns identity. + * Validates the session token (auth schema) and returns the user as identity. */ export async function authenticateAccounts( request: Request, - accountsDb: AccountsDB, + auth: AuthStore, ): Promise { const token = extractBearerToken(request); if (!token) { @@ -134,8 +134,8 @@ export async function authenticateAccounts( }; } - const sessionResult = await accountsDb.validateSession(token); - if (!sessionResult) { + const session = await auth.validateSession(token); + if (!session) { debug("accounts auth failed: invalid or expired session"); return { ok: false, @@ -143,12 +143,16 @@ export async function authenticateAccounts( }; } - debug("accounts auth succeeded", { identityId: sessionResult.identity.id }); + debug("accounts auth succeeded", { userId: session.userId }); return { ok: true, context: { type: "accounts", - identity: sessionResult.identity, + identity: { + id: session.userId, + email: session.email, + name: session.name, + }, }, }; } diff --git a/packages/server/router.test.ts b/packages/server/router.test.ts index 3651abe..37e1e55 100644 --- a/packages/server/router.test.ts +++ b/packages/server/router.test.ts @@ -1,7 +1,9 @@ import { describe, expect, mock, test } from "bun:test"; import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { SQL } from "bun"; +import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; import { createRouter } from "./router"; @@ -15,6 +17,12 @@ function createMockContext(): ServerContext { } as unknown as AccountsDB, accountsSql: {} as SQL, engineSql: {} as SQL, + db: {} as Sql, + auth: { + validateSession: mock(() => Promise.resolve(null)), + } as unknown as AuthStore, + authSchema: "auth", + coreSchema: "core", embeddingConfig: { provider: "openai", model: "text-embedding-3-small", diff --git a/packages/server/router.ts b/packages/server/router.ts index 98bb645..22daed8 100644 --- a/packages/server/router.ts +++ b/packages/server/router.ts @@ -1,6 +1,7 @@ import type { ServerContext } from "./context"; import { type AuthHandlerContext, + deviceApproveHandler, deviceCodeHandler, deviceTokenHandler, deviceVerifyGetHandler, @@ -123,18 +124,23 @@ export function createRouter(ctx: ServerContext): Router { accountsDb, accountsSql, engineSql, + db, + auth, + authSchema, + coreSchema, embeddingConfig, apiBaseUrl, serverVersion, minClientVersion, } = ctx; - // Auth handler context for device flow endpoints + // Auth handler context for the device flow + OAuth endpoints const authCtx: AuthHandlerContext = { - db: accountsDb, + auth, + db, + authSchema, + coreSchema, baseUrl: apiBaseUrl, - engineSql, - serverVersion, }; // Wrap an RPC handler with the X-Client-Version check, so requests from @@ -176,19 +182,20 @@ export function createRouter(ctx: ServerContext): Router { const accountsRpcHandler = createRpcHandler( accountsMethods, async (request) => { - const auth = await authenticateAccounts(request, accountsDb); - if (!auth.ok) { - return auth.error; + const result = await authenticateAccounts(request, auth); + if (!result.ok) { + return result.error; } - // TypeScript narrows auth.context to AuthContext after ok check + // TypeScript narrows result.context to AuthContext after ok check // We know it's AccountsAuthContext since we called authenticateAccounts - const ctx = auth.context; - if (ctx.type !== "accounts") { + const authContext = result.context; + if (authContext.type !== "accounts") { throw new Error("Unexpected auth context type"); } return { db: accountsDb, - identity: ctx.identity, + auth, + identity: authContext.identity, engineSql, serverVersion, }; @@ -246,6 +253,13 @@ export function createRouter(ctx: ServerContext): Router { handler: (req) => deviceVerifyPostHandler(req, authCtx), }, + // OAuth Device Flow - User approves/denies after OAuth (consent step) + { + method: "POST", + pattern: "/api/v1/auth/device/approve", + handler: (req) => deviceApproveHandler(req, authCtx), + }, + // OAuth Callback - Provider redirects here after user authorizes { method: "GET", diff --git a/packages/server/rpc/accounts/engine.integration.test.ts b/packages/server/rpc/accounts/engine.integration.test.ts index bbbfc05..9fd108e 100644 --- a/packages/server/rpc/accounts/engine.integration.test.ts +++ b/packages/server/rpc/accounts/engine.integration.test.ts @@ -75,6 +75,9 @@ function createContext(identity: Identity): HandlerContext { return { request: new Request("http://localhost"), db: accountsDb, + // engine handlers are legacy (AccountsDB) and never touch the auth store; + // a stub object satisfies the assertAccountsRpcContext guard. + auth: {} as unknown, identity, engineSql, serverVersion: SERVER_VERSION, diff --git a/packages/server/rpc/accounts/me.test.ts b/packages/server/rpc/accounts/me.test.ts index d2d52a1..3f9e637 100644 --- a/packages/server/rpc/accounts/me.test.ts +++ b/packages/server/rpc/accounts/me.test.ts @@ -1,27 +1,39 @@ /** * Unit tests for identity/me RPC handlers. * - * Uses mocked AccountsDB to test handler logic in isolation. + * Uses a mocked AuthStore to test handler logic in isolation. */ import { describe, expect, mock, test } from "bun:test"; +import type { User } from "@memory.build/auth"; import type { HandlerContext } from "../types"; import { meMethods } from "./me"; +const authUser: User = { + id: "019d694f-79f6-7595-8faf-b70b01c11f98", + email: "alice@example.com", + name: "Alice", + emailVerified: true, + image: null, + createdAt: new Date("2026-01-15T00:00:00.000Z"), + updatedAt: null, +}; + function createMockContext( - dbOverrides: Record = {}, + authOverrides: Record = {}, ): HandlerContext { return { request: new Request("http://localhost"), - db: { - getIdentityByEmail: mock(() => Promise.resolve(null)), - ...dbOverrides, + // Legacy AccountsDB stub — present only to satisfy the context guard. + db: {}, + auth: { + getUser: mock(() => Promise.resolve(authUser)), + getUserByEmail: mock(() => Promise.resolve(null)), + ...authOverrides, }, identity: { - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "alice@example.com", - name: "Alice", - createdAt: new Date("2026-01-15T00:00:00.000Z"), - updatedAt: null, + id: authUser.id, + email: authUser.email, + name: authUser.name, }, engineSql: mock(() => {}) as unknown, serverVersion: "0.1.1", @@ -33,7 +45,7 @@ function createMockContext( // ============================================================================= describe("me.get", () => { - test("returns the authenticated identity", async () => { + test("returns the authenticated user", async () => { const handler = meMethods.get("me.get")?.handler; if (!handler) throw new Error("me.get handler not found"); @@ -44,10 +56,22 @@ describe("me.get", () => { name: string; }; - expect(result.id).toBe("019d694f-79f6-7595-8faf-b70b01c11f98"); + expect(result.id).toBe(authUser.id); expect(result.email).toBe("alice@example.com"); expect(result.name).toBe("Alice"); }); + + test("looks up the user by the session identity id", async () => { + const handler = meMethods.get("me.get")?.handler; + if (!handler) throw new Error("me.get handler not found"); + + const getUser = mock(() => Promise.resolve(authUser)); + const context = createMockContext({ getUser }); + + await handler({}, context); + + expect(getUser).toHaveBeenCalledWith(authUser.id); + }); }); // ============================================================================= @@ -59,16 +83,18 @@ describe("identity.getByEmail", () => { const handler = meMethods.get("identity.getByEmail")?.handler; if (!handler) throw new Error("identity.getByEmail handler not found"); - const identity = { + const bob: User = { id: "019d694f-79f6-7595-8faf-b70b01c11f99", email: "bob@example.com", name: "Bob", + emailVerified: true, + image: null, createdAt: new Date("2026-01-15T00:00:00.000Z"), updatedAt: null, }; const context = createMockContext({ - getIdentityByEmail: mock(() => Promise.resolve(identity)), + getUserByEmail: mock(() => Promise.resolve(bob)), }); const result = (await handler({ email: "bob@example.com" }, context)) as { @@ -86,7 +112,7 @@ describe("identity.getByEmail", () => { if (!handler) throw new Error("identity.getByEmail handler not found"); const context = createMockContext({ - getIdentityByEmail: mock(() => Promise.resolve(null)), + getUserByEmail: mock(() => Promise.resolve(null)), }); const result = (await handler( @@ -97,15 +123,15 @@ describe("identity.getByEmail", () => { expect(result.identity).toBeNull(); }); - test("passes email to db lookup", async () => { + test("passes email to the auth-store lookup", async () => { const handler = meMethods.get("identity.getByEmail")?.handler; if (!handler) throw new Error("identity.getByEmail handler not found"); - const getIdentityByEmail = mock(() => Promise.resolve(null)); - const context = createMockContext({ getIdentityByEmail }); + const getUserByEmail = mock(() => Promise.resolve(null)); + const context = createMockContext({ getUserByEmail }); await handler({ email: "test@example.com" }, context); - expect(getIdentityByEmail).toHaveBeenCalledWith("test@example.com"); + expect(getUserByEmail).toHaveBeenCalledWith("test@example.com"); }); }); diff --git a/packages/server/rpc/accounts/me.ts b/packages/server/rpc/accounts/me.ts index 01f3142..f0539ab 100644 --- a/packages/server/rpc/accounts/me.ts +++ b/packages/server/rpc/accounts/me.ts @@ -1,10 +1,7 @@ /** - * Accounts RPC me methods. - * - * Implements: - * - me.get: Get the current authenticated identity + * Accounts RPC me/identity methods (auth-schema backed). */ -import type { Identity } from "@memory.build/accounts"; +import type { User } from "@memory.build/auth"; import type { IdentityGetByEmailParams, IdentityGetByEmailResult, @@ -17,58 +14,43 @@ import { } from "@memory.build/protocol/accounts/identity"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; -import { assertAccountsRpcContext } from "./types"; +import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; -/** - * Convert an Identity to a serializable response. - */ -function toIdentityResponse(identity: Identity): IdentityResponse { +function toIdentityResponse(user: User): IdentityResponse { return { - id: identity.id, - email: identity.email, - name: identity.name, - createdAt: identity.createdAt.toISOString(), - updatedAt: identity.updatedAt?.toISOString() ?? null, + id: user.id, + email: user.email, + name: user.name, + createdAt: user.createdAt.toISOString(), + updatedAt: user.updatedAt?.toISOString() ?? null, }; } -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * me.get - Get the current authenticated identity. - */ +/** me.get — the current authenticated user. */ async function meGet( _params: MeGetParams, context: HandlerContext, ): Promise { assertAccountsRpcContext(context); - // Identity is already available from authentication - no DB lookup needed - return toIdentityResponse(context.identity); + const { auth, identity } = context as AccountsRpcContext; + const user = await auth.getUser(identity.id); + if (!user) { + throw new Error("Authenticated user not found"); + } + return toIdentityResponse(user); } -/** - * identity.getByEmail - Look up an identity by email address. - */ +/** identity.getByEmail — look up a user by email. */ async function identityGetByEmail( params: IdentityGetByEmailParams, context: HandlerContext, ): Promise { assertAccountsRpcContext(context); - const { db } = context as import("./types").AccountsRpcContext; - - const identity = await db.getIdentityByEmail(params.email); - return { identity: identity ? toIdentityResponse(identity) : null }; + const { auth } = context as AccountsRpcContext; + const user = await auth.getUserByEmail(params.email); + return { identity: user ? toIdentityResponse(user) : null }; } -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the me/identity methods registry. - */ export const meMethods = buildRegistry() .register("me.get", meGetParams, meGet) .register("identity.getByEmail", identityGetByEmailParams, identityGetByEmail) diff --git a/packages/server/rpc/accounts/org.integration.test.ts b/packages/server/rpc/accounts/org.integration.test.ts index c11a6ce..eb2d00f 100644 --- a/packages/server/rpc/accounts/org.integration.test.ts +++ b/packages/server/rpc/accounts/org.integration.test.ts @@ -39,6 +39,9 @@ function createContext(identity: Identity): HandlerContext { return { request: new Request("http://localhost"), db: accountsDb, + // org handlers are legacy (AccountsDB) and never touch the auth store; + // a stub object satisfies the assertAccountsRpcContext guard. + auth: {} as unknown, identity, // org.update never touches engineSql; a stub that satisfies the // assertAccountsRpcContext type guard (typeof === "function") is enough. diff --git a/packages/server/rpc/accounts/session.ts b/packages/server/rpc/accounts/session.ts index 63dbc0c..64ee4da 100644 --- a/packages/server/rpc/accounts/session.ts +++ b/packages/server/rpc/accounts/session.ts @@ -27,9 +27,9 @@ async function sessionRevoke( context: HandlerContext, ): Promise<{ revoked: boolean }> { assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; + const { auth, identity } = context as AccountsRpcContext; - const count = await db.deleteSessionsByIdentity(identity.id); + const count = await auth.deleteSessionsByUser(identity.id); return { revoked: count > 0 }; } diff --git a/packages/server/rpc/accounts/types.ts b/packages/server/rpc/accounts/types.ts index ab6f494..4081b19 100644 --- a/packages/server/rpc/accounts/types.ts +++ b/packages/server/rpc/accounts/types.ts @@ -4,6 +4,7 @@ * Extends the base HandlerContext with accounts-specific fields. */ import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { SQL } from "bun"; import type { Identity } from "../../middleware/authenticate"; import type { HandlerContext } from "../types"; @@ -20,9 +21,11 @@ import type { HandlerContext } from "../types"; * Authentication middleware populates these fields via OAuth session validation. */ export interface AccountsRpcContext extends HandlerContext { - /** AccountsDB instance */ + /** Legacy AccountsDB instance (org/engine/member/invitation, until Phase 5) */ db: AccountsDB; - /** Authenticated identity */ + /** Auth store (auth schema): me/session/identity */ + auth: AuthStore; + /** Authenticated user (session-validated) */ identity: Identity; /** SQL connection to the engine database */ engineSql: SQL; @@ -40,6 +43,9 @@ export function isAccountsRpcContext( "db" in ctx && typeof ctx.db === "object" && ctx.db !== null && + "auth" in ctx && + typeof ctx.auth === "object" && + ctx.auth !== null && "identity" in ctx && typeof ctx.identity === "object" && ctx.identity !== null && diff --git a/packages/server/server.integration.test.ts b/packages/server/server.integration.test.ts index f9b0fd2..55c308f 100644 --- a/packages/server/server.integration.test.ts +++ b/packages/server/server.integration.test.ts @@ -1,7 +1,9 @@ import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { SQL } from "bun"; +import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; import { MAX_BODY_SIZE } from "./middleware/size-limit"; @@ -16,19 +18,29 @@ function createMockContext(): ServerContext { accountsDb: { validateSession: mock(() => Promise.resolve(null)), getEngineBySlug: mock(() => Promise.resolve(null)), - // Device auth operations for auth endpoint tests - create: mock(() => Promise.resolve({})), - getByDeviceCode: mock(() => Promise.resolve(null)), - getByUserCode: mock(() => Promise.resolve(null)), - getByOAuthState: mock(() => Promise.resolve(null)), - updateLastPoll: mock(() => Promise.resolve(null)), - authorize: mock(() => Promise.resolve(false)), - deny: mock(() => Promise.resolve(false)), - delete: mock(() => Promise.resolve(false)), - deleteExpired: mock(() => Promise.resolve(0)), } as unknown as AccountsDB, accountsSql: {} as SQL, engineSql: {} as SQL, + db: {} as Sql, + auth: { + // Session validation: no session → accounts RPC stays 401. + validateSession: mock(() => Promise.resolve(null)), + // Device flow operations exercised by the auth endpoint tests. + createDeviceAuth: mock((_provider: string) => + Promise.resolve({ + deviceCode: "test-device-code", + userCode: "WXYZ-2345", + oauthState: "test-oauth-state", + expiresIn: 900, + }), + ), + // Unknown device codes poll as expired. + pollDevice: mock(() => + Promise.resolve({ status: "expired", userId: null }), + ), + } as unknown as AuthStore, + authSchema: "auth", + coreSchema: "core", embeddingConfig: { provider: "openai", model: "text-embedding-3-small", diff --git a/packages/server/wiring.test.ts b/packages/server/wiring.test.ts index d6a8346..4c4e8ba 100644 --- a/packages/server/wiring.test.ts +++ b/packages/server/wiring.test.ts @@ -10,8 +10,10 @@ import { describe, expect, mock, test } from "bun:test"; import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { SQL } from "bun"; +import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; import type { EngineInfo } from "./middleware/authenticate"; @@ -22,17 +24,25 @@ import { createRouter } from "./router"; // ============================================================================= function createMockAccountsDb(overrides?: { - validateSession?: ReturnType; getEngineBySlug?: ReturnType; }): AccountsDB { return { - validateSession: - overrides?.validateSession ?? mock(() => Promise.resolve(null)), getEngineBySlug: overrides?.getEngineBySlug ?? mock(() => Promise.resolve(null)), } as unknown as AccountsDB; } +function createMockAuth(overrides?: { + validateSession?: ReturnType; + getUser?: ReturnType; +}): AuthStore { + return { + validateSession: + overrides?.validateSession ?? mock(() => Promise.resolve(null)), + getUser: overrides?.getUser ?? mock(() => Promise.resolve(null)), + } as unknown as AuthStore; +} + /** * Create a mock SQL that has enough methods to not throw, but returns * no results. This allows testing wiring without a real database. @@ -62,6 +72,10 @@ function createMockContext(overrides?: Partial): ServerContext { accountsDb: createMockAccountsDb(), accountsSql: {} as SQL, engineSql: createMockEngineSql(), + db: {} as Sql, + auth: createMockAuth(), + authSchema: "auth", + coreSchema: "core", embeddingConfig: { provider: "openai", model: "text-embedding-3-small", @@ -173,11 +187,24 @@ describe("Server-Database Wiring", () => { }; const ctx = createMockContext({ - accountsDb: createMockAccountsDb({ + auth: createMockAuth({ validateSession: mock(() => Promise.resolve({ - session: { id: "session-1", identityId: mockIdentity.id }, - identity: mockIdentity, + sessionId: "session-1", + userId: mockIdentity.id, + email: mockIdentity.email, + name: mockIdentity.name, + expiresAt: new Date("2026-12-31T00:00:00Z"), + }), + ), + getUser: mock(() => + Promise.resolve({ + id: mockIdentity.id, + email: mockIdentity.email, + name: mockIdentity.name, + emailVerified: true, + createdAt: new Date("2026-01-01T00:00:00Z"), + updatedAt: null, }), ), }), From 621e7dfcbff494591af2a9d8997f03a70b821ee0 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 18:53:46 +0200 Subject: [PATCH 040/156] feat(server): add /api/v1/memory/rpc endpoint + authenticateSpace (4C-0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stand up the new-model memory RPC foundation alongside the untouched legacy /api/v1/engine/rpc. - authenticateSpace: resolves principal + space + treeAccess for the memory endpoint. Uniform X-Me-Space header (both modes); principal from core.validateApiKey (agent) or auth.validateSession (human); core.buildTreeAccess is the single authorization gate (empty → 403). Optional 400 when an api key's embedded slug ≠ the header. - SpaceRpcContext + guards; empty memoryMethods registry (methods land in 4C-1/4C-2). - inject coreStore via ServerContext (built once in index.ts, like authStore). - wire POST /api/v1/memory/rpc in the router. - integration test: both credential modes resolve correctly; failure paths (401 missing/invalid creds, 400 missing header / slug mismatch, 403 non-member). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/server/context.ts | 3 + packages/server/index.ts | 5 + .../authenticate-space.integration.test.ts | 215 ++++++++++++++++++ .../server/middleware/authenticate-space.ts | 178 +++++++++++++++ packages/server/middleware/index.ts | 7 + packages/server/router.test.ts | 2 + packages/server/router.ts | 34 ++- packages/server/rpc/index.ts | 8 +- packages/server/rpc/memory/index.ts | 17 ++ packages/server/rpc/memory/types.ts | 60 +++++ packages/server/server.integration.test.ts | 2 + packages/server/wiring.test.ts | 2 + 12 files changed, 530 insertions(+), 3 deletions(-) create mode 100644 packages/server/middleware/authenticate-space.integration.test.ts create mode 100644 packages/server/middleware/authenticate-space.ts create mode 100644 packages/server/rpc/memory/index.ts create mode 100644 packages/server/rpc/memory/types.ts diff --git a/packages/server/context.ts b/packages/server/context.ts index 2c37878..7c00b80 100644 --- a/packages/server/context.ts +++ b/packages/server/context.ts @@ -1,6 +1,7 @@ import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { CoreStore } from "@memory.build/engine/core"; import type { SQL } from "bun"; import type { Sql } from "postgres"; @@ -19,6 +20,8 @@ export interface ServerContext { db: Sql; /** Auth store (auth schema): me/session/identity/device + OAuth accounts */ auth: AuthStore; + /** Core control-plane store (core schema): spaces/principals/grants/api-keys */ + core: CoreStore; /** The auth schema name */ authSchema: string; /** The core control-plane schema name */ diff --git a/packages/server/index.ts b/packages/server/index.ts index 2da225d..ba9ac08 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -8,6 +8,7 @@ import { migrateCore, } from "@memory.build/database"; import type { EmbeddingConfig } from "@memory.build/embedding"; +import { coreStore } from "@memory.build/engine/core"; import { discoverEngineSchemas, slugToSchema, @@ -380,6 +381,9 @@ const accountsDb = createAccountsDB(accountsSql, accountsSchema); // Auth store (auth schema) on the new-model postgres.js pool. const auth = authStore(db, authSchema); +// Core control-plane store (core schema) on the same pool. +const core = coreStore(db, coreSchema); + // ============================================================================= // Database Bootstrap & Migrations (blocking — server won't serve until current) // ============================================================================= @@ -470,6 +474,7 @@ const serverContext: ServerContext = { engineSql, db, auth, + core, authSchema, coreSchema, embeddingConfig, diff --git a/packages/server/middleware/authenticate-space.integration.test.ts b/packages/server/middleware/authenticate-space.integration.test.ts new file mode 100644 index 0000000..b5632a1 --- /dev/null +++ b/packages/server/middleware/authenticate-space.integration.test.ts @@ -0,0 +1,215 @@ +// Integration test for space authentication (authenticateSpace). +// +// Stands up auth + core schemas and the space DB in one database, provisions a +// user (auth identity + core principal + space + owner grant), then exercises +// the session and api-key credential modes plus the failure paths. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/middleware/authenticate-space.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import { core as engineCore } from "@memory.build/engine"; +import postgres, { type Sql } from "postgres"; +import { provisionUser } from "../provision"; +import { authenticateSpace, SPACE_HEADER } from "./authenticate-space"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; +const email = () => `space_${crypto.randomUUID().slice(0, 8)}@example.com`; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +// The deps authenticateSpace needs; bound to the test schemas. +function deps() { + return { + core: engineCore.coreStore(sql, coreSchema), + auth: authStore(sql, authSchema), + db: sql, + }; +} + +/** Build a request with optional bearer token + X-Me-Space header. */ +function req(opts: { token?: string; space?: string }): Request { + const headers: Record = {}; + if (opts.token) headers.Authorization = `Bearer ${opts.token}`; + if (opts.space) headers[SPACE_HEADER] = opts.space; + return new Request("http://localhost/api/v1/memory/rpc", { + method: "POST", + headers, + }); +} + +// Provision a user + space and return its slug, the user id, and a session token. +async function provision() { + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: email(), + name: "Tester", + provider: "github", + accountId: crypto.randomUUID(), + }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + const { token } = await authStore(sql, authSchema).createSession(r.userId); + return { ...r, token }; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +test("session: member with owner grant resolves space + treeAccess", async () => { + const p = await provision(); + const result = await authenticateSpace( + req({ token: p.token, space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.context.space.id).toBe(p.spaceId); + expect(result.context.principalId).toBe(p.userId); + expect(result.context.apiKeyId).toBeNull(); + expect(result.context.treeAccess).toContainEqual({ + tree_path: engineCore.ROOT_PATH, + access: engineCore.ACCESS.owner, + }); + } +}); + +test("api key: agent of the space resolves with apiKeyId set", async () => { + const p = await provision(); + const core = engineCore.coreStore(sql, coreSchema); + + const agentId = await core.createAgent(p.userId, `agent-${rand()}`); + await core.addPrincipalToSpace(p.spaceId, agentId); + await core.grantTreeAccess( + p.spaceId, + agentId, + engineCore.ROOT_PATH, + engineCore.ACCESS.read, + ); + const key = await core.createApiKey(agentId, "ci"); + const fullKey = engineCore.formatApiKey( + p.spaceSlug, + key.lookupId, + key.secret, + ); + + const result = await authenticateSpace( + req({ token: fullKey, space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.context.principalId).toBe(agentId); + expect(result.context.apiKeyId).not.toBeNull(); + expect(result.context.treeAccess.length).toBeGreaterThan(0); + } +}); + +test("missing Authorization → 401", async () => { + const result = await authenticateSpace( + req({ space: "abcdef012345" }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(401); +}); + +test("missing X-Me-Space → 400", async () => { + const p = await provision(); + const result = await authenticateSpace(req({ token: p.token }), deps()); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(400); +}); + +test("unknown space → 401", async () => { + const p = await provision(); + const result = await authenticateSpace( + req({ token: p.token, space: "zzzzzz999999" }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(401); +}); + +test("invalid session token → 401", async () => { + const p = await provision(); + const result = await authenticateSpace( + req({ token: "not-a-real-session-token", space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(401); +}); + +test("api key slug ≠ header → 400 (SPACE_MISMATCH)", async () => { + const p = await provision(); + const other = await provision(); + const core = engineCore.coreStore(sql, coreSchema); + const agentId = await core.createAgent(p.userId, `agent-${rand()}`); + await core.grantTreeAccess( + p.spaceId, + agentId, + engineCore.ROOT_PATH, + engineCore.ACCESS.read, + ); + const key = await core.createApiKey(agentId, "ci"); + // key minted for p's slug, but header points at a different space + const fullKey = engineCore.formatApiKey( + p.spaceSlug, + key.lookupId, + key.secret, + ); + const result = await authenticateSpace( + req({ token: fullKey, space: other.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(400); +}); + +test("session: member of another space has no grant here → 403", async () => { + const a = await provision(); + const b = await provision(); + // b's session against a's space — b has no grant in a's space. + const result = await authenticateSpace( + req({ token: b.token, space: a.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(403); +}); diff --git a/packages/server/middleware/authenticate-space.ts b/packages/server/middleware/authenticate-space.ts new file mode 100644 index 0000000..2640116 --- /dev/null +++ b/packages/server/middleware/authenticate-space.ts @@ -0,0 +1,178 @@ +/** + * Authentication for the space memory RPC (`/api/v1/memory/rpc`). + * + * Resolves an authenticated principal and a target space into the access set + * (`treeAccess`) that the space SQL functions consume. Two credential modes, + * discriminated by whether the bearer token parses as an api key: + * + * - api key (agent): `me...` — validated against core. + * - session (human): an opaque session token — validated against auth. + * + * The space is always selected by the `X-Me-Space` header (uniform for both + * modes). `core.buildTreeAccess(principalId, space.id)` is the single + * authorization gate: a principal with no grants in the space (including an api + * key minted for a different space) resolves to an empty set and is denied. + */ +import type { AuthStore } from "@memory.build/auth"; +import { slugToSchema } from "@memory.build/database"; +import { + type CoreStore, + parseApiKey, + type Space, + type TreeAccess, +} from "@memory.build/engine/core"; +import { type SpaceStore, spaceStore } from "@memory.build/engine/space"; +import { debug, span } from "@pydantic/logfire-node"; +import type { Sql } from "postgres"; +import { error, forbidden, unauthorized } from "../util/response"; +import { extractBearerToken } from "./authenticate"; + +/** Header that selects which space a session / api-key request targets. */ +export const SPACE_HEADER = "X-Me-Space"; + +/** + * The authenticated principal + resolved space for a memory RPC request. + */ +export interface SpaceAuthContext { + type: "space"; + /** Space data-plane store bound to the `me_` schema. */ + store: SpaceStore; + /** Core control-plane store (shared; used by the management methods). */ + core: CoreStore; + /** The resolved space. */ + space: Space; + /** Authenticated principal id (user id for sessions, agent id for api keys). */ + principalId: string; + /** Api key id when authenticated by api key; null for sessions. */ + apiKeyId: string | null; + /** The principal's effective grants in this space — the access gate. */ + treeAccess: TreeAccess; +} + +export type SpaceAuthResult = + | { ok: true; context: SpaceAuthContext } + | { ok: false; error: Response }; + +export interface SpaceAuthDeps { + /** Core control-plane store (on the new-model pool). */ + core: CoreStore; + /** Auth store (auth schema) for session validation. */ + auth: AuthStore; + /** New-model pool — used to bind the per-space data-plane store. */ + db: Sql; +} + +/** + * Authenticate a memory RPC request and resolve its space access set. + */ +export async function authenticateSpace( + request: Request, + deps: SpaceAuthDeps, +): Promise { + return span("auth.space", { + attributes: { "auth.type": "space" }, + callback: () => authenticateSpaceInner(request, deps), + }); +} + +async function authenticateSpaceInner( + request: Request, + deps: SpaceAuthDeps, +): Promise { + const { core, auth, db } = deps; + + // 1. Bearer token (a session token or an api key). + const token = extractBearerToken(request); + if (!token) { + debug("space auth failed: missing Authorization header"); + return { + ok: false, + error: unauthorized("Missing or invalid Authorization header"), + }; + } + + // 2. Space slug — always from the X-Me-Space header (uniform for both modes). + const slug = request.headers.get(SPACE_HEADER); + if (!slug) { + debug("space auth failed: missing X-Me-Space header"); + return { + ok: false, + error: error(`Missing ${SPACE_HEADER} header`, 400, "MISSING_SPACE"), + }; + } + + // 3. Resolve the space (shared step). Generic 401 to avoid space enumeration. + const space = await core.getSpace(slug); + if (!space) { + debug("space auth failed: unknown space", { slug }); + return { ok: false, error: unauthorized("Invalid credentials") }; + } + + // 4. Resolve the principal — the only line that differs between modes. + const parsed = parseApiKey(token); + let principalId: string; + let apiKeyId: string | null; + + if (parsed) { + // The api key embeds its own slug; assert it matches the header so a + // misrouted key gives a clear error rather than a confusing 403 below. + if (parsed.spaceSlug !== slug) { + debug("space auth failed: api key slug != header", { + slug, + keySlug: parsed.spaceSlug, + }); + return { + ok: false, + error: error( + `API key is not valid for space ${slug}`, + 400, + "SPACE_MISMATCH", + ), + }; + } + const validated = await core.validateApiKey(parsed.lookupId, parsed.secret); + if (!validated) { + debug("space auth failed: invalid api key"); + return { ok: false, error: unauthorized("Invalid credentials") }; + } + principalId = validated.memberId; + apiKeyId = validated.apiKeyId; + } else { + const session = await auth.validateSession(token); + if (!session) { + debug("space auth failed: invalid or expired session"); + return { ok: false, error: unauthorized("Invalid credentials") }; + } + principalId = session.userId; + apiKeyId = null; + } + + // 5. The single membership / authorization gate. An empty set means the + // principal has no grants in this space (incl. a wrong-space api key) — deny. + const treeAccess = await core.buildTreeAccess(principalId, space.id); + if (treeAccess.length === 0) { + debug("space auth failed: no access in space", { slug, principalId }); + return { ok: false, error: forbidden("No access to this space") }; + } + + // 6. Bind the data-plane store to this space's schema. + const store = spaceStore(db, slugToSchema(space.slug)); + + debug("space auth succeeded", { + slug, + principalId, + byApiKey: apiKeyId !== null, + }); + return { + ok: true, + context: { + type: "space", + store, + core, + space, + principalId, + apiKeyId, + treeAccess, + }, + }; +} diff --git a/packages/server/middleware/index.ts b/packages/server/middleware/index.ts index dafa8e0..d5285d0 100644 --- a/packages/server/middleware/index.ts +++ b/packages/server/middleware/index.ts @@ -11,6 +11,13 @@ export { extractBearerToken, type Identity, } from "./authenticate"; +export { + authenticateSpace, + SPACE_HEADER, + type SpaceAuthContext, + type SpaceAuthDeps, + type SpaceAuthResult, +} from "./authenticate-space"; export { checkClientVersion } from "./client-version"; export { checkSizeLimit, diff --git a/packages/server/router.test.ts b/packages/server/router.test.ts index 37e1e55..8fc96e9 100644 --- a/packages/server/router.test.ts +++ b/packages/server/router.test.ts @@ -2,6 +2,7 @@ import { describe, expect, mock, test } from "bun:test"; import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { CoreStore } from "@memory.build/engine/core"; import type { SQL } from "bun"; import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; @@ -21,6 +22,7 @@ function createMockContext(): ServerContext { auth: { validateSession: mock(() => Promise.resolve(null)), } as unknown as AuthStore, + core: {} as unknown as CoreStore, authSchema: "auth", coreSchema: "core", embeddingConfig: { diff --git a/packages/server/router.ts b/packages/server/router.ts index 22daed8..7cfa722 100644 --- a/packages/server/router.ts +++ b/packages/server/router.ts @@ -14,8 +14,14 @@ import { authenticateAccounts, authenticateEngine, } from "./middleware/authenticate"; +import { authenticateSpace } from "./middleware/authenticate-space"; import { checkClientVersion } from "./middleware/client-version"; -import { accountsMethods, createRpcHandler, engineMethods } from "./rpc"; +import { + accountsMethods, + createRpcHandler, + engineMethods, + memoryMethods, +} from "./rpc"; import { notFound } from "./util/response"; /** @@ -126,6 +132,7 @@ export function createRouter(ctx: ServerContext): Router { engineSql, db, auth, + core, authSchema, coreSchema, embeddingConfig, @@ -202,6 +209,24 @@ export function createRouter(ctx: ServerContext): Router { }, ); + // Memory RPC (new model): authenticate principal + space, provide space context + const memoryRpcHandler = createRpcHandler(memoryMethods, async (request) => { + const result = await authenticateSpace(request, { core, auth, db }); + if (!result.ok) { + return result.error; + } + const spaceContext = result.context; + return { + store: spaceContext.store, + core: spaceContext.core, + space: spaceContext.space, + principalId: spaceContext.principalId, + apiKeyId: spaceContext.apiKeyId, + treeAccess: spaceContext.treeAccess, + embeddingConfig, + }; + }); + /** * Application routes. * @@ -280,6 +305,13 @@ export function createRouter(ctx: ServerContext): Router { pattern: "/api/v1/engine/rpc", handler: withClientVersionCheck(engineRpcHandler), }, + + // Memory RPC (new model: space data-plane + management) + { + method: "POST", + pattern: "/api/v1/memory/rpc", + handler: withClientVersionCheck(memoryRpcHandler), + }, ]; /** diff --git a/packages/server/rpc/index.ts b/packages/server/rpc/index.ts index 787b087..525f00a 100644 --- a/packages/server/rpc/index.ts +++ b/packages/server/rpc/index.ts @@ -6,7 +6,6 @@ export { engineMethods, isEngineContext, } from "./engine"; - // Errors export { APP_ERROR_CODES, @@ -24,9 +23,14 @@ export { parseError, RPC_ERROR_CODES, } from "./errors"; - // Handler export { createRpcHandler, handleRpcRequest } from "./handler"; +export { + assertSpaceRpcContext, + isSpaceRpcContext, + memoryMethods, + type SpaceRpcContext, +} from "./memory"; // Registry export { buildRegistry, diff --git a/packages/server/rpc/memory/index.ts b/packages/server/rpc/memory/index.ts new file mode 100644 index 0000000..3609e87 --- /dev/null +++ b/packages/server/rpc/memory/index.ts @@ -0,0 +1,17 @@ +/** + * Memory RPC method registry — served at `/api/v1/memory/rpc`. + * + * The new-model replacement for the engine RPC: memory data-plane methods + * (spaceStore) and space management methods (coreStore). Methods are added in + * Phase 4C-1 (memory.*) and 4C-2 (user/grant/owner/role/apiKey.*); for now the + * endpoint + authenticateSpace plumbing exists with an empty registry. + */ +import { buildRegistry } from "../registry"; + +export { + assertSpaceRpcContext, + isSpaceRpcContext, + type SpaceRpcContext, +} from "./types"; + +export const memoryMethods = buildRegistry().build(); diff --git a/packages/server/rpc/memory/types.ts b/packages/server/rpc/memory/types.ts new file mode 100644 index 0000000..da95982 --- /dev/null +++ b/packages/server/rpc/memory/types.ts @@ -0,0 +1,60 @@ +/** + * Memory RPC context types. + * + * The context for `/api/v1/memory/rpc` — populated by authenticateSpace. Memory + * (data-plane) methods use `store` + `treeAccess`; management (control-plane) + * methods use `core` + `space`. + */ +import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { CoreStore, Space, TreeAccess } from "@memory.build/engine/core"; +import type { SpaceStore } from "@memory.build/engine/space"; +import type { HandlerContext } from "../types"; + +export interface SpaceRpcContext extends HandlerContext { + /** Space data-plane store bound to the `me_` schema. */ + store: SpaceStore; + /** Core control-plane store (management methods). */ + core: CoreStore; + /** The resolved space. */ + space: Space; + /** Authenticated principal id (user id for sessions, agent id for api keys). */ + principalId: string; + /** Api key id when authenticated by api key; null for sessions. */ + apiKeyId: string | null; + /** The principal's effective grants in this space — the access gate. */ + treeAccess: TreeAccess; + /** Embedding config for semantic search (optional). */ + embeddingConfig?: EmbeddingConfig; +} + +/** + * Type guard for the memory RPC context. + */ +export function isSpaceRpcContext(ctx: HandlerContext): ctx is SpaceRpcContext { + return ( + "store" in ctx && + typeof ctx.store === "object" && + ctx.store !== null && + "core" in ctx && + typeof ctx.core === "object" && + ctx.core !== null && + "space" in ctx && + typeof ctx.space === "object" && + ctx.space !== null && + "principalId" in ctx && + typeof ctx.principalId === "string" && + "treeAccess" in ctx && + Array.isArray(ctx.treeAccess) + ); +} + +/** + * Assert that context is a SpaceRpcContext, throwing if not. + */ +export function assertSpaceRpcContext( + ctx: HandlerContext, +): asserts ctx is SpaceRpcContext { + if (!isSpaceRpcContext(ctx)) { + throw new Error("Space context not initialized (authentication required)"); + } +} diff --git a/packages/server/server.integration.test.ts b/packages/server/server.integration.test.ts index 55c308f..a18c29a 100644 --- a/packages/server/server.integration.test.ts +++ b/packages/server/server.integration.test.ts @@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { CoreStore } from "@memory.build/engine/core"; import type { SQL } from "bun"; import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; @@ -39,6 +40,7 @@ function createMockContext(): ServerContext { Promise.resolve({ status: "expired", userId: null }), ), } as unknown as AuthStore, + core: {} as unknown as CoreStore, authSchema: "auth", coreSchema: "core", embeddingConfig: { diff --git a/packages/server/wiring.test.ts b/packages/server/wiring.test.ts index 4c4e8ba..812baa4 100644 --- a/packages/server/wiring.test.ts +++ b/packages/server/wiring.test.ts @@ -12,6 +12,7 @@ import { describe, expect, mock, test } from "bun:test"; import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { CoreStore } from "@memory.build/engine/core"; import type { SQL } from "bun"; import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; @@ -74,6 +75,7 @@ function createMockContext(overrides?: Partial): ServerContext { engineSql: createMockEngineSql(), db: {} as Sql, auth: createMockAuth(), + core: {} as unknown as CoreStore, authSchema: "auth", coreSchema: "core", embeddingConfig: { From 9cb9993323e778f841d303046164c81be5f01c99 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 19:06:54 +0200 Subject: [PATCH 041/156] feat(server): wire memory.* RPC onto spaceStore (4C-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the memory data-plane methods at /api/v1/memory/rpc against the space store, keeping the wire protocol unchanged and adapting in the handler. - memory.create/batchCreate/get/update/delete/search/tree/move/deleteTree/ countTree → spaceStore, gated by the request's treeAccess. - search mapping: fulltext→bm25, semantic→embed→vec, grep→regexp, tree→ltree, meta→metaContains, temporal within/overlaps/contains→space ranges, semanticThreshold→maxVecDist (1−t), weights→hybrid weights; fulltext+semantic routes to hybridSearch, otherwise search. - memory.tree builds an lquery from {tree, levels} and reshapes list_tree. - lossy by design (4C): createdBy is null, search total is the row count, orderBy ignored. Postgres errors mapped: 42501→FORBIDDEN, 22023/22P02→ VALIDATION_ERROR. - add spaceStore.withTransaction (mirrors coreStore) so batchCreate is atomic. - fix space search_memory: the filter-only arm emitted `-1 as score` (integer) against a float8 return column → `(-1)::float8`. - integration test covers CRUD, tree/levels, move/deleteTree dryRun, fulltext + filter search, grep-alone rejection, and write-access denial. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../space/migrate/idempotent/002_search.sql | 3 +- packages/engine/space/db.ts | 11 + packages/server/rpc/memory/index.ts | 10 +- .../rpc/memory/memory.integration.test.ts | 333 +++++++++++++ packages/server/rpc/memory/memory.ts | 469 ++++++++++++++++++ 5 files changed, 818 insertions(+), 8 deletions(-) create mode 100644 packages/server/rpc/memory/memory.integration.test.ts create mode 100644 packages/server/rpc/memory/memory.ts diff --git a/packages/database/space/migrate/idempotent/002_search.sql b/packages/database/space/migrate/idempotent/002_search.sql index 6da0335..0b6bbbc 100644 --- a/packages/database/space/migrate/idempotent/002_search.sql +++ b/packages/database/space/migrate/idempotent/002_search.sql @@ -77,7 +77,8 @@ begin ); end if; else - _score = $sql$, -1 as score$sql$; + -- no ranking arm: constant score, typed float8 to match the return column + _score = $sql$, (-1)::float8 as score$sql$; _order_by = $sql$order by m.id$sql$; end case; diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts index fdb9e3f..f61a7fa 100644 --- a/packages/engine/space/db.ts +++ b/packages/engine/space/db.ts @@ -63,6 +63,9 @@ export interface SpaceStore { treeAccess: TreeAccess, options: HybridSearchOptions, ): Promise; + + /** Run operations atomically against the same transaction. */ + withTransaction(fn: (store: SpaceStore) => Promise): Promise; } function mapMemory(row: Record): Memory { @@ -243,5 +246,13 @@ export function spaceStore(sql: Sql, schema: string): SpaceStore { )`; return rows.map(mapSearchItem); }, + + async withTransaction( + fn: (store: SpaceStore) => Promise, + ): Promise { + return sql.begin((tx) => + fn(spaceStore(tx as unknown as Sql, schema)), + ) as Promise; + }, }; } diff --git a/packages/server/rpc/memory/index.ts b/packages/server/rpc/memory/index.ts index 3609e87..593721f 100644 --- a/packages/server/rpc/memory/index.ts +++ b/packages/server/rpc/memory/index.ts @@ -2,16 +2,12 @@ * Memory RPC method registry — served at `/api/v1/memory/rpc`. * * The new-model replacement for the engine RPC: memory data-plane methods - * (spaceStore) and space management methods (coreStore). Methods are added in - * Phase 4C-1 (memory.*) and 4C-2 (user/grant/owner/role/apiKey.*); for now the - * endpoint + authenticateSpace plumbing exists with an empty registry. + * (spaceStore) and, in 4C-2, space management methods (coreStore). Memory.* + * methods are wired here; management methods are added in Phase 4C-2. */ -import { buildRegistry } from "../registry"; - +export { memoryMethods } from "./memory"; export { assertSpaceRpcContext, isSpaceRpcContext, type SpaceRpcContext, } from "./types"; - -export const memoryMethods = buildRegistry().build(); diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts new file mode 100644 index 0000000..dd69634 --- /dev/null +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -0,0 +1,333 @@ +// Integration test for the memory RPC data-plane handlers (4C-1). +// +// Provisions a space + owner, then drives the memory.* handlers through the +// registry against a real space schema (me_). Semantic search is not +// exercised (embeddings are generated by the worker — Phase 4D); fulltext +// (bm25) and filter search are. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/memory/memory.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import { core as engineCore, space as engineSpace } from "@memory.build/engine"; +import type { TreeAccess } from "@memory.build/engine/core"; +import type { SpaceStore } from "@memory.build/engine/space"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import { provisionUser } from "../../provision"; +import type { HandlerContext } from "../types"; +import { memoryMethods } from "./memory"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +// Per-test space context. +let store: SpaceStore; +let treeAccess: TreeAccess; +let space: { id: string; slug: string }; +let principalId: string; + +function call( + method: string, + params: unknown, + ta: TreeAccess = treeAccess, +): Promise { + const registered = memoryMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/memory/rpc"), + store, + core: engineCore.coreStore(sql, coreSchema), + space, + principalId, + apiKeyId: null, + treeAccess: ta, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +/** Assert a handler call rejects with a specific AppError code. */ +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +// Fresh space per test so trees don't bleed across cases. +beforeEach(async () => { + const core = engineCore.coreStore(sql, coreSchema); + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: `mem_${crypto.randomUUID().slice(0, 8)}@example.com`, + name: "Owner", + provider: "github", + accountId: crypto.randomUUID(), + }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + store = engineSpace.spaceStore(sql, `me_${r.spaceSlug}`); + treeAccess = await core.buildTreeAccess(r.userId, r.spaceId); + space = { id: r.spaceId, slug: r.spaceSlug }; + principalId = r.userId; +}); + +test("create → get round-trips content/tree/meta and createdBy is null", async () => { + const created = await call<{ id: string; createdBy: string | null }>( + "memory.create", + { content: "hello world", tree: "notes.work", meta: { tag: "a" } }, + ); + expect(created.createdBy).toBeNull(); + + const got = await call<{ + id: string; + content: string; + tree: string; + meta: Record; + hasEmbedding: boolean; + }>("memory.get", { id: created.id }); + expect(got.content).toBe("hello world"); + expect(got.tree).toBe("notes.work"); + expect(got.meta).toEqual({ tag: "a" }); + expect(got.hasEmbedding).toBe(false); +}); + +test("create with temporal round-trips as {start,end}", async () => { + const created = await call<{ id: string }>("memory.create", { + content: "temporal one", + temporal: { start: "2024-01-01T00:00:00Z", end: "2024-01-02T00:00:00Z" }, + }); + const got = await call<{ temporal: { start: string; end: string } | null }>( + "memory.get", + { id: created.id }, + ); + expect(got.temporal).toEqual({ + start: "2024-01-01T00:00:00.000Z", + end: "2024-01-02T00:00:00.000Z", + }); +}); + +test("update patches fields", async () => { + const created = await call<{ id: string }>("memory.create", { + content: "before", + tree: "a", + }); + const updated = await call<{ content: string; tree: string }>( + "memory.update", + { id: created.id, content: "after", tree: "a.b" }, + ); + expect(updated.content).toBe("after"); + expect(updated.tree).toBe("a.b"); +}); + +test("delete removes; get then NOT_FOUND", async () => { + const created = await call<{ id: string }>("memory.create", { + content: "doomed", + }); + const res = await call<{ deleted: boolean }>("memory.delete", { + id: created.id, + }); + expect(res.deleted).toBe(true); + await expectAppError(call("memory.get", { id: created.id }), "NOT_FOUND"); +}); + +test("get / delete unknown id → NOT_FOUND", async () => { + const ghost = crypto.randomUUID(); + await expectAppError(call("memory.get", { id: ghost }), "NOT_FOUND"); + await expectAppError(call("memory.delete", { id: ghost }), "NOT_FOUND"); +}); + +test("batchCreate inserts all and is retrievable", async () => { + const res = await call<{ ids: string[] }>("memory.batchCreate", { + memories: [ + { content: "one", tree: "batch" }, + { content: "two", tree: "batch" }, + { content: "three", tree: "batch.sub" }, + ], + }); + expect(res.ids).toHaveLength(3); + const count = await call<{ count: number }>("memory.countTree", { + tree: "batch", + }); + expect(count.count).toBe(3); +}); + +test("tree returns descendant node counts under a path", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "x", tree: "root.a" }, + { content: "y", tree: "root.a.deep" }, + { content: "z", tree: "root.b" }, + ], + }); + const res = await call<{ nodes: { path: string; count: number }[] }>( + "memory.tree", + { tree: "root" }, + ); + const byPath = Object.fromEntries(res.nodes.map((n) => [n.path, n.count])); + expect(byPath["root.a"]).toBe(2); + expect(byPath["root.a.deep"]).toBe(1); + expect(byPath["root.b"]).toBe(1); + // the base path itself is excluded + expect(byPath.root).toBeUndefined(); +}); + +test("tree respects levels depth limit", async () => { + await call("memory.batchCreate", { + memories: [{ content: "deep", tree: "t.a.b.c" }], + }); + const res = await call<{ nodes: { path: string }[] }>("memory.tree", { + tree: "t", + levels: 1, + }); + const paths = res.nodes.map((n) => n.path); + expect(paths).toContain("t.a"); + expect(paths).not.toContain("t.a.b"); +}); + +test("move relocates a subtree (dryRun counts without moving)", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "m1", tree: "src.x" }, + { content: "m2", tree: "src.y" }, + ], + }); + const dry = await call<{ count: number }>("memory.move", { + source: "src", + destination: "dst", + dryRun: true, + }); + expect(dry.count).toBe(2); + // still under src + expect( + (await call<{ count: number }>("memory.countTree", { tree: "src" })).count, + ).toBe(2); + + const moved = await call<{ count: number }>("memory.move", { + source: "src", + destination: "dst", + }); + expect(moved.count).toBe(2); + expect( + (await call<{ count: number }>("memory.countTree", { tree: "src" })).count, + ).toBe(0); + expect( + (await call<{ count: number }>("memory.countTree", { tree: "dst" })).count, + ).toBe(2); +}); + +test("deleteTree removes a subtree (dryRun counts without deleting)", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "d1", tree: "gone.a" }, + { content: "d2", tree: "gone.b" }, + ], + }); + const dry = await call<{ count: number }>("memory.deleteTree", { + tree: "gone", + dryRun: true, + }); + expect(dry.count).toBe(2); + const del = await call<{ count: number }>("memory.deleteTree", { + tree: "gone", + }); + expect(del.count).toBe(2); + expect( + (await call<{ count: number }>("memory.countTree", { tree: "gone" })).count, + ).toBe(0); +}); + +test("search: fulltext (bm25) finds matching content", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "the quick brown fox", tree: "s" }, + { content: "lazy dogs sleep", tree: "s" }, + ], + }); + const res = await call<{ results: { content: string }[]; total: number }>( + "memory.search", + { fulltext: "fox" }, + ); + expect(res.total).toBeGreaterThanOrEqual(1); + expect(res.results.some((r) => r.content.includes("fox"))).toBe(true); +}); + +test("search: tree filter only (no ranking) returns matches", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "in scope", tree: "scope.a" }, + { content: "out of scope", tree: "other" }, + ], + }); + const res = await call<{ results: { tree: string }[] }>("memory.search", { + tree: "scope", + }); + expect(res.results.length).toBe(1); + expect(res.results[0]?.tree).toBe("scope.a"); +}); + +test("search: grep alone is rejected", async () => { + await expectAppError( + call("memory.search", { grep: "anything" }), + "VALIDATION_ERROR", + ); +}); + +test("search: semantic without embedding config → EMBEDDING_NOT_CONFIGURED", async () => { + await expectAppError( + call("memory.search", { semantic: "meaning" }), + "EMBEDDING_NOT_CONFIGURED", + ); +}); + +test("create without write access → FORBIDDEN", async () => { + // read-only on the root path + const readOnly: TreeAccess = [ + { tree_path: "", access: engineCore.ACCESS.read }, + ]; + await expectAppError( + call("memory.create", { content: "nope", tree: "x" }, readOnly), + "FORBIDDEN", + ); +}); diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts new file mode 100644 index 0000000..fa1b2f4 --- /dev/null +++ b/packages/server/rpc/memory/memory.ts @@ -0,0 +1,469 @@ +/** + * Memory RPC data-plane methods (new model) — served at `/api/v1/memory/rpc`. + * + * Adapts the stable memory.* wire protocol onto the space data-plane store + * (spaceStore). The wire is unchanged from the legacy engine RPC; the mapping + * is handler-local. Lossy by design (see Phase 4C): `createdBy` is always null + * (the space model has no per-memory creator), search `total` is the returned + * row count, and `orderBy` is ignored (ranked search is score-desc only). + */ +import { generateEmbedding } from "@memory.build/embedding"; +import { ACCESS, ROOT_PATH } from "@memory.build/engine/core"; +import type { + SearchResultItem, + Memory as SpaceMemory, +} from "@memory.build/engine/space"; +import type { + MemoryBatchCreateParams, + MemoryBatchCreateResult, + MemoryCountTreeParams, + MemoryCountTreeResult, + MemoryCreateParams, + MemoryDeleteParams, + MemoryDeleteResult, + MemoryDeleteTreeParams, + MemoryDeleteTreeResult, + MemoryGetParams, + MemoryMoveParams, + MemoryMoveResult, + MemoryResponse, + MemorySearchParams, + MemorySearchResult, + MemoryTreeParams, + MemoryTreeResult, + MemoryUpdateParams, +} from "@memory.build/protocol/engine/memory"; +import { + memoryBatchCreateParams, + memoryCountTreeParams, + memoryCreateParams, + memoryDeleteParams, + memoryDeleteTreeParams, + memoryGetParams, + memoryMoveParams, + memorySearchParams, + memoryTreeParams, + memoryUpdateParams, +} from "@memory.build/protocol/engine/memory"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +// ============================================================================= +// Helpers +// ============================================================================= + +/** + * Translate a space SQL error into an AppError. The space functions raise + * `insufficient_privilege` (42501) on access violations and + * `invalid_parameter_value` (22023) / `invalid_text_representation` (22P02) + * on malformed input; everything else propagates as an internal error. + */ +function mapSpaceError(e: unknown): never { + const code = (e as { code?: string }).code; + if (code === "42501") { + throw new AppError("FORBIDDEN", "Insufficient tree access"); + } + if (code === "22023" || code === "22P02") { + throw new AppError( + "VALIDATION_ERROR", + e instanceof Error ? e.message : "Invalid parameter", + ); + } + throw e instanceof Error ? e : new Error(String(e)); +} + +/** Run a space-store call, mapping its SQL errors to AppErrors. */ +async function guard(fn: () => Promise): Promise { + try { + return await fn(); + } catch (e) { + return mapSpaceError(e); + } +} + +/** + * Format a wire temporal `{start, end?}` into a PostgreSQL tstzrange string. + * Point-in-time (no end / end == start) → `[t,t]`; otherwise `[start,end)`. + * Mirrors the legacy engine's tstzrange formatting. + */ +function formatTemporal( + t: { start: string; end?: string | null } | null | undefined, +): string | undefined { + if (!t) return undefined; + const start = t.start; + const end = t.end ?? start; + return start === end ? `[${start},${end}]` : `[${start},${end})`; +} + +/** + * Parse a PostgreSQL tstzrange string into a wire `{start, end}` (ISO), + * normalizing the timestamps. Mirrors the legacy engine's parser. + */ +function parseTemporal( + range: string | null, +): { start: string; end: string } | null { + if (!range) return null; + const m = range.match(/[[(]"?([^",]+)"?,"?([^",\])]+)"?[\])]/); + if (!m) return null; + const [, start, end] = m; + if (!start || !end) return null; + return { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + }; +} + +/** ltree depth (label count); root ("") is 0. */ +function nlevel(path: string): number { + return path === "" ? 0 : path.split(".").length; +} + +function toMemoryResponse(m: SpaceMemory): MemoryResponse { + return { + id: m.id, + content: m.content, + meta: m.meta, + tree: m.tree, + temporal: parseTemporal(m.temporal), + hasEmbedding: m.hasEmbedding, + createdAt: m.createdAt.toISOString(), + // The space model does not track a per-memory creator (4C decision). + createdBy: null, + updatedAt: m.updatedAt?.toISOString() ?? null, + }; +} + +/** + * Map the wire temporal filter (contains | overlaps | within — mutually + * exclusive) onto the space search's temporal range params. A `contains` + * point becomes an inclusive point-range overlap (true iff the memory's range + * spans the instant). + */ +function mapTemporalFilter(tf: MemorySearchParams["temporal"]): { + temporalWithin?: string; + temporalOverlaps?: string; +} { + if (!tf) return {}; + if (tf.within) { + return { temporalWithin: `[${tf.within.start},${tf.within.end})` }; + } + if (tf.overlaps) { + return { temporalOverlaps: `[${tf.overlaps.start},${tf.overlaps.end})` }; + } + if (tf.contains) { + return { temporalOverlaps: `[${tf.contains},${tf.contains}]` }; + } + return {}; +} + +// ============================================================================= +// Method Handlers +// ============================================================================= + +/** memory.create */ +async function memoryCreate( + params: MemoryCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const id = await guard(() => + store.createMemory(treeAccess, { + id: params.id ?? undefined, + content: params.content, + meta: params.meta ?? undefined, + tree: params.tree ?? ROOT_PATH, + temporal: formatTemporal(params.temporal), + }), + ); + const memory = await store.getMemory(treeAccess, id); + if (!memory) { + throw new AppError("INTERNAL_ERROR", "Created memory could not be read"); + } + return toMemoryResponse(memory); +} + +/** memory.batchCreate — atomic across the batch. */ +async function memoryBatchCreate( + params: MemoryBatchCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const ids = await guard(() => + store.withTransaction(async (tx) => { + const out: string[] = []; + for (const m of params.memories) { + out.push( + await tx.createMemory(treeAccess, { + id: m.id ?? undefined, + content: m.content, + meta: m.meta ?? undefined, + tree: m.tree ?? ROOT_PATH, + temporal: formatTemporal(m.temporal), + }), + ); + } + return out; + }), + ); + return { ids }; +} + +/** memory.get */ +async function memoryGet( + params: MemoryGetParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const memory = await guard(() => store.getMemory(treeAccess, params.id)); + if (!memory) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + return toMemoryResponse(memory); +} + +/** memory.update */ +async function memoryUpdate( + params: MemoryUpdateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const patch: { + content?: string; + meta?: Record; + tree?: string; + temporal?: string | null; + } = {}; + if (params.content !== undefined && params.content !== null) { + patch.content = params.content; + } + if (params.meta !== undefined && params.meta !== null) { + patch.meta = params.meta; + } + if (params.tree !== undefined && params.tree !== null) { + patch.tree = params.tree; + } + if (params.temporal !== undefined) { + patch.temporal = + params.temporal === null + ? null + : (formatTemporal(params.temporal) ?? null); + } + + const ok = await guard(() => store.patchMemory(treeAccess, params.id, patch)); + if (!ok) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + const memory = await store.getMemory(treeAccess, params.id); + if (!memory) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + return toMemoryResponse(memory); +} + +/** memory.delete */ +async function memoryDelete( + params: MemoryDeleteParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const deleted = await guard(() => store.deleteMemory(treeAccess, params.id)); + if (!deleted) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + return { deleted }; +} + +/** memory.search — hybrid (fulltext+semantic) or single-arm / filter-only. */ +async function memorySearch( + params: MemorySearchParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess, embeddingConfig } = context as SpaceRpcContext; + + // Generate the query embedding for semantic search. + let vec: number[] | undefined; + if (params.semantic) { + if (!embeddingConfig) { + throw new AppError( + "EMBEDDING_NOT_CONFIGURED", + "Semantic search requires embedding configuration. Set EMBEDDING_API_KEY.", + ); + } + try { + vec = (await generateEmbedding(params.semantic, embeddingConfig)) + .embedding; + } catch (error) { + throw new AppError( + "EMBEDDING_FAILED", + `Failed to generate embedding: ${error instanceof Error ? error.message : "Unknown error"}`, + ); + } + } + + const bm25 = params.fulltext ?? undefined; + + // grep alone would force a full table scan — require another indexed filter. + if ( + params.grep && + !params.fulltext && + !params.semantic && + !params.tree && + !params.meta && + !params.temporal + ) { + throw new AppError( + "VALIDATION_ERROR", + "grep cannot be used alone (full table scan). Combine with semantic, fulltext, tree, meta, or temporal.", + ); + } + + // semanticThreshold is a cosine similarity (0..1); the space search filters by + // cosine distance (= 1 - similarity), and only when a vector is present. + const maxVecDist = + vec && params.semanticThreshold != null + ? 1 - params.semanticThreshold + : undefined; + + const filters = { + ltree: params.tree ?? undefined, + metaContains: params.meta ?? undefined, + regexp: params.grep ?? undefined, + ...mapTemporalFilter(params.temporal), + }; + const limit = params.limit ?? 10; + + let items: SearchResultItem[]; + if (bm25 && vec) { + items = await guard(() => + store.hybridSearch(treeAccess, { + bm25, + vec, + maxVecDist, + candidateLimit: params.candidateLimit, + fulltextWeight: params.weights?.fulltext, + semanticWeight: params.weights?.semantic, + limit, + ...filters, + }), + ); + } else { + items = await guard(() => + store.search(treeAccess, { bm25, vec, maxVecDist, limit, ...filters }), + ); + } + + return { + results: items.map((item) => ({ + ...toMemoryResponse(item), + score: item.score, + })), + total: items.length, + limit, + }; +} + +/** memory.tree — node counts under a path, down to `levels` depth. */ +async function memoryTree( + params: MemoryTreeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const base = params.tree ?? ""; + // `a.b.*` matches a.b and everything under it; `*` matches all paths. + const lquery = base === "" ? "*" : `${base}.*`; + const entries = await guard(() => store.listTree(treeAccess, lquery)); + + const baseDepth = nlevel(base); + const nodes = entries + .filter((e) => { + const depth = nlevel(e.tree); + // strict descendants of the base path (exclude the base and its ancestors) + if (depth <= baseDepth) return false; + if (params.levels !== undefined && depth - baseDepth > params.levels) { + return false; + } + return true; + }) + .map((e) => ({ path: e.tree, count: e.count })); + + return { nodes }; +} + +/** memory.move */ +async function memoryMove( + params: MemoryMoveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const count = await guard(() => + store.moveTree( + treeAccess, + params.source, + params.destination, + params.dryRun ?? false, + ), + ); + return { count }; +} + +/** memory.deleteTree */ +async function memoryDeleteTree( + params: MemoryDeleteTreeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const count = await guard(() => + store.deleteTree(treeAccess, params.tree, params.dryRun ?? false), + ); + return { count }; +} + +/** memory.countTree */ +async function memoryCountTree( + params: MemoryCountTreeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const count = await guard(() => + store.countTree(treeAccess, { tree: params.tree }, ACCESS.read), + ); + return { count }; +} + +// ============================================================================= +// Registry +// ============================================================================= + +export const memoryMethods = buildRegistry() + .register("memory.create", memoryCreateParams, memoryCreate) + .register("memory.batchCreate", memoryBatchCreateParams, memoryBatchCreate) + .register("memory.get", memoryGetParams, memoryGet) + .register("memory.update", memoryUpdateParams, memoryUpdate) + .register("memory.delete", memoryDeleteParams, memoryDelete) + .register("memory.search", memorySearchParams, memorySearch) + .register("memory.tree", memoryTreeParams, memoryTree) + .register("memory.move", memoryMoveParams, memoryMove) + .register("memory.deleteTree", memoryDeleteTreeParams, memoryDeleteTree) + .register("memory.countTree", memoryCountTreeParams, memoryCountTree) + .build(); From 599f2b75b59582221c1e8027fc2d09e6d507276c Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 19:37:55 +0200 Subject: [PATCH 042/156] feat(core): add management read/list/delete primitives (4C-2a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the core SQL + coreStore with the read/mutate primitives the space management RPC (4C-2b) needs, beyond the create/grant/membership functions that already existed. - principals: get_user_by_name, list_space_groups, rename_principal (agents/ groups only — a user's name is its identity email), delete_principal (cascades). - membership: list_space_members (Model 2 — unions direct principal_space and via-group members, deduped, each flagged `direct`), list_group_members, list_groups_for_member. - grants: list_tree_access_grants (raw grant rows; named to avoid confusion with build_tree_access, which resolves the effective access set). - api keys: get_api_key, list_api_keys (secret never returned), delete_api_key (revoke ≡ delete; no soft-revoke state). - coreStore methods + types (SpaceMember/Group/GroupMember/GroupMembership/ TreeGrant/ApiKeyInfo); core integration test covers all of the above. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../core/migrate/idempotent/005_principal.sql | 98 ++++++++++ .../migrate/idempotent/006_membership.sql | 103 +++++++++++ .../core/migrate/idempotent/007_grant.sql | 27 +++ .../core/migrate/idempotent/008_api_key.sql | 68 +++++++ packages/engine/core/core.integration.test.ts | 167 ++++++++++++++++++ packages/engine/core/db.ts | 160 +++++++++++++++++ packages/engine/core/index.ts | 6 + packages/engine/core/types.ts | 60 +++++++ 8 files changed, 689 insertions(+) create mode 100644 packages/engine/core/core.integration.test.ts diff --git a/packages/database/core/migrate/idempotent/005_principal.sql b/packages/database/core/migrate/idempotent/005_principal.sql index 5d41932..e27ef4f 100644 --- a/packages/database/core/migrate/idempotent/005_principal.sql +++ b/packages/database/core/migrate/idempotent/005_principal.sql @@ -74,3 +74,101 @@ as $func$ $func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; + +------------------------------------------------------------------------------- +-- get_user_by_name +-- Resolve a global user (kind 'u') by name. User names are globally unique +-- (citext), so this returns at most one row. +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_user_by_name +( _name text +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, space_id uuid +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.kind, p.name::text, p.owner_id, p.space_id, p.created_at, p.updated_at + from {{schema}}.principal p + where p.kind = 'u' + and p.name = _name::citext +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_space_groups +-- All groups belonging to a space (groups are space-scoped via space_id). +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_space_groups +( _space_id uuid +) +returns table +( id uuid +, name text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.name::text, p.created_at, p.updated_at + from {{schema}}.principal p + where p.kind = 'g' + and p.space_id = _space_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- rename_principal +-- Rename an agent or group. Users are intentionally excluded: a user's name is +-- its email — the global identity handle that mirrors auth.users — so changing +-- it is an account concern, not a space-management one. Returns true if an +-- agent/group with this id was renamed. Name uniqueness is enforced by the +-- principal table indexes. +------------------------------------------------------------------------------- +create or replace function {{schema}}.rename_principal +( _id uuid +, _name text +) +returns bool +as $func$ + with u as + ( + update {{schema}}.principal + set name = _name::citext + where id = _id + and kind in ('a', 'g') -- never rename users (kind 'u') + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_principal +-- Delete a principal row. Foreign keys cascade: a user's agents (owner_id), +-- its space memberships, group memberships, tree-access grants, and api keys +-- all go with it. Returns true if a row was deleted. +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_principal +( _id uuid +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.principal + where id = _id + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/006_membership.sql b/packages/database/core/migrate/idempotent/006_membership.sql index 00986e3..eb2f2eb 100644 --- a/packages/database/core/migrate/idempotent/006_membership.sql +++ b/packages/database/core/migrate/idempotent/006_membership.sql @@ -98,3 +98,106 @@ as $func$ $func$ language sql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; + +------------------------------------------------------------------------------- +-- list_space_members +-- Principals that belong to a space, deduplicated: either added directly +-- (principal_space) or reached through a group in the space (group_member) — +-- group membership confers space access, so both count. `direct` is true when +-- the principal has a direct membership row; `admin` is its direct-membership +-- admin flag (false for group-only members). Optional kind filter +-- ('u' | 'a' | 'g'); null returns all. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_space_members +( _space_id uuid +, _kind text default null +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, direct bool +, admin bool +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + with mem as + ( + -- directly added to the space + select ps.principal_id as id, true as direct, ps.admin as admin + from {{schema}}.principal_space ps + where ps.space_id = _space_id + union all + -- reached through a group belonging to the space + select gm.member_id as id, false as direct, false as admin + from {{schema}}.group_member gm + where gm.space_id = _space_id + ) + , agg as + ( + select id, bool_or(direct) as direct, bool_or(admin) as admin + from mem + group by id + ) + select p.id, p.kind, p.name::text, p.owner_id, agg.direct, agg.admin, p.created_at, p.updated_at + from agg + join {{schema}}.principal p on p.id = agg.id + where (_kind is null or p.kind = _kind) + order by p.kind, p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_group_members +-- Members (users / agents) of a group within a space, with the admin flag. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_group_members +( _space_id uuid +, _group_id uuid +) +returns table +( member_id uuid +, kind text +, name text +, admin bool +, created_at timestamptz +) +as $func$ + select gm.member_id, p.kind, p.name::text, gm.admin, gm.created_at + from {{schema}}.group_member gm + join {{schema}}.principal p on p.id = gm.member_id + where gm.space_id = _space_id + and gm.group_id = _group_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_groups_for_member +-- Groups within a space that a member (user / agent) belongs to, with the +-- admin flag. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_groups_for_member +( _space_id uuid +, _member_id uuid +) +returns table +( group_id uuid +, name text +, admin bool +, created_at timestamptz +) +as $func$ + select gm.group_id, p.name::text, gm.admin, gm.created_at + from {{schema}}.group_member gm + join {{schema}}.principal p on p.id = gm.group_id + where gm.space_id = _space_id + and gm.member_id = _member_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/007_grant.sql b/packages/database/core/migrate/idempotent/007_grant.sql index f9194e1..25ea58b 100644 --- a/packages/database/core/migrate/idempotent/007_grant.sql +++ b/packages/database/core/migrate/idempotent/007_grant.sql @@ -44,3 +44,30 @@ as $func$ $func$ language sql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; + +------------------------------------------------------------------------------- +-- list_tree_access_grants +-- The grant rows in a space, optionally for a single principal. (Owner listing +-- is this filtered to access = 3 by the caller.) Distinct from build_tree_access, +-- which resolves a member's *effective* access set; this lists the raw grants. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_tree_access_grants +( _space_id uuid +, _principal_id uuid default null +) +returns table +( principal_id uuid +, tree_path text +, access int +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select t.principal_id, t.tree_path::text, t.access, t.created_at, t.updated_at + from {{schema}}.tree_access t + where t.space_id = _space_id + and (_principal_id is null or t.principal_id = _principal_id) + order by t.principal_id, t.tree_path +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/008_api_key.sql b/packages/database/core/migrate/idempotent/008_api_key.sql index 3fb7990..f01cdda 100644 --- a/packages/database/core/migrate/idempotent/008_api_key.sql +++ b/packages/database/core/migrate/idempotent/008_api_key.sql @@ -41,3 +41,71 @@ as $func$ $func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; + +------------------------------------------------------------------------------- +-- get_api_key +-- Key metadata by id (never the secret). +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_api_key +( _id uuid +) +returns table +( id uuid +, member_id uuid +, lookup_id text +, name text +, created_at timestamptz +, expires_at timestamptz +) +as $func$ + select k.id, k.member_id, k.lookup_id, k.name, k.created_at, k.expires_at + from {{schema}}.api_key k + where k.id = _id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_api_keys +-- A member's keys (never the secret), newest first. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_api_keys +( _member_id uuid +) +returns table +( id uuid +, member_id uuid +, lookup_id text +, name text +, created_at timestamptz +, expires_at timestamptz +) +as $func$ + select k.id, k.member_id, k.lookup_id, k.name, k.created_at, k.expires_at + from {{schema}}.api_key k + where k.member_id = _member_id + order by k.created_at desc +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_api_key +-- Hard-delete a key by id. Returns true if a row was deleted. (There is no +-- soft-revoke state; revoke and delete are the same operation.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_api_key +( _id uuid +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.api_key + where id = _id + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts new file mode 100644 index 0000000..2709d88 --- /dev/null +++ b/packages/engine/core/core.integration.test.ts @@ -0,0 +1,167 @@ +// Integration test for the core control-plane store additions (4C-2a): +// principal listing/rename/delete, group membership listing, grant listing, +// and api-key read/delete. Runs against a real core schema. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/engine/core/core.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { migrateCore } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { type CoreStore, coreStore } from "./db"; +import { ACCESS } from "./types"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let coreSchema: string; +let core: CoreStore; + +// Fresh space + owner user per test. +let spaceId: string; +let userId: string; +let userName: string; + +async function v7(): Promise { + const [row] = await sql`select uuidv7() as id`; + return row?.id as string; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + await migrateCore(sql, { schema: coreSchema }); + core = coreStore(sql, coreSchema); +}); + +afterAll(async () => { + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + spaceId = await core.createSpace(rand(12), "Test Space"); + userId = await v7(); + userName = `user_${rand(8)}@example.com`; + await core.createUser(userId, userName); + await core.addPrincipalToSpace(spaceId, userId, true); +}); + +test("getUserByName resolves a global user", async () => { + const u = await core.getUserByName(userName); + expect(u?.id).toBe(userId); + expect(u?.kind).toBe("u"); + expect(await core.getUserByName("nobody@example.com")).toBeNull(); +}); + +test("renamePrincipal refuses to rename users", async () => { + expect(await core.renamePrincipal(userId, "new@example.com")).toBe(false); + // the user's name is unchanged + expect((await core.getPrincipal(userId))?.name).toBe(userName); +}); + +test("listSpaceMembers lists direct members with admin flag and kind filter", async () => { + const all = await core.listSpaceMembers(spaceId); + expect(all).toHaveLength(1); + expect(all[0]?.id).toBe(userId); + expect(all[0]?.direct).toBe(true); + expect(all[0]?.admin).toBe(true); + + expect(await core.listSpaceMembers(spaceId, "u")).toHaveLength(1); + expect(await core.listSpaceMembers(spaceId, "g")).toHaveLength(0); +}); + +test("listSpaceMembers includes group-only members (flagged direct=false)", async () => { + // a second user who is NOT added to the space directly, only via a group + const groupOnlyId = await v7(); + await core.createUser(groupOnlyId, `grouponly_${rand(8)}@example.com`); + const groupId = await core.createGroup(spaceId, "team"); + await core.addGroupMember(spaceId, groupId, groupOnlyId); + + const members = await core.listSpaceMembers(spaceId, "u"); + const byId = Object.fromEntries(members.map((m) => [m.id, m])); + // owner is a direct member + expect(byId[userId]?.direct).toBe(true); + // the group-only user shows up as a member, flagged direct=false + expect(byId[groupOnlyId]).toBeDefined(); + expect(byId[groupOnlyId]?.direct).toBe(false); + expect(byId[groupOnlyId]?.admin).toBe(false); +}); + +test("agents appear as space members of kind 'a'", async () => { + const agentId = await core.createAgent(userId, `agent-${rand(6)}`); + await core.addPrincipalToSpace(spaceId, agentId); + const agents = await core.listSpaceMembers(spaceId, "a"); + expect(agents).toHaveLength(1); + expect(agents[0]?.id).toBe(agentId); + expect(agents[0]?.ownerId).toBe(userId); +}); + +test("groups: create, list, rename, members, delete", async () => { + const groupId = await core.createGroup(spaceId, "eng"); + let groups = await core.listSpaceGroups(spaceId); + expect(groups.map((g) => g.name)).toContain("eng"); + + expect(await core.renamePrincipal(groupId, "engineering")).toBe(true); + groups = await core.listSpaceGroups(spaceId); + expect(groups.find((g) => g.id === groupId)?.name).toBe("engineering"); + + await core.addGroupMember(spaceId, groupId, userId, true); + const members = await core.listGroupMembers(spaceId, groupId); + expect(members).toHaveLength(1); + expect(members[0]?.memberId).toBe(userId); + expect(members[0]?.admin).toBe(true); + + const forMember = await core.listGroupsForMember(spaceId, userId); + expect(forMember.map((g) => g.groupId)).toContain(groupId); + + expect(await core.removeGroupMember(spaceId, groupId, userId)).toBe(true); + expect(await core.listGroupMembers(spaceId, groupId)).toHaveLength(0); + + expect(await core.deletePrincipal(groupId)).toBe(true); + expect(await core.listSpaceGroups(spaceId)).toHaveLength(0); +}); + +test("listTreeAccessGrants returns grants; filterable by principal", async () => { + await core.grantTreeAccess(spaceId, userId, "a.b", ACCESS.write); + await core.grantTreeAccess(spaceId, userId, "c", ACCESS.owner); + + const all = await core.listTreeAccessGrants(spaceId); + const paths = all.map((g) => g.treePath).sort(); + expect(paths).toEqual(["a.b", "c"]); + expect(all.find((g) => g.treePath === "c")?.access).toBe(ACCESS.owner); + + const forUser = await core.listTreeAccessGrants(spaceId, userId); + expect(forUser).toHaveLength(2); + + expect(await core.removeTreeAccessGrant(spaceId, userId, "a.b")).toBe(true); + expect(await core.listTreeAccessGrants(spaceId)).toHaveLength(1); +}); + +test("api keys: create, get, list, delete (no secret leaked)", async () => { + const key = await core.createApiKey(userId, "ci"); + expect(key.secret).toBeTruthy(); + + const got = await core.getApiKey(key.id); + expect(got?.id).toBe(key.id); + expect(got?.memberId).toBe(userId); + expect(got?.lookupId).toBe(key.lookupId); + expect(got?.name).toBe("ci"); + // metadata only — no secret field on ApiKeyInfo + expect((got as unknown as Record).secret).toBeUndefined(); + + const list = await core.listApiKeys(userId); + expect(list.map((k) => k.id)).toContain(key.id); + + expect(await core.deleteApiKey(key.id)).toBe(true); + expect(await core.getApiKey(key.id)).toBeNull(); + expect(await core.listApiKeys(userId)).toHaveLength(0); +}); diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index 4dcecd0..6ab4831 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -3,11 +3,17 @@ import type { Sql } from "postgres"; import { generateLookupId, generateSecret, hashApiKeySecret } from "./api-key"; import type { AccessLevel, + ApiKeyInfo, CreatedApiKey, + Group, + GroupMember, + GroupMembership, Principal, PrincipalKind, Space, + SpaceMember, TreeAccess, + TreeGrant, ValidatedApiKey, } from "./types"; @@ -26,6 +32,19 @@ export interface CoreStore { createAgent(ownerId: string, name: string, id?: string): Promise; createGroup(spaceId: string, name: string, id?: string): Promise; getPrincipal(id: string): Promise; + /** Resolve a global user (kind 'u') by name (email). */ + getUserByName(name: string): Promise; + /** Rename an agent or group (never a user — its name is its identity email). */ + renamePrincipal(id: string, name: string): Promise; + deletePrincipal(id: string): Promise; + + /** Principals in a space — directly or via a group (each flagged `direct`). */ + listSpaceMembers( + spaceId: string, + kind?: PrincipalKind, + ): Promise; + /** Groups belonging to a space. */ + listSpaceGroups(spaceId: string): Promise; addPrincipalToSpace( spaceId: string, @@ -47,6 +66,13 @@ export interface CoreStore { groupId: string, memberId: string, ): Promise; + /** Members (users / agents) of a group within a space. */ + listGroupMembers(spaceId: string, groupId: string): Promise; + /** Groups within a space that a member belongs to. */ + listGroupsForMember( + spaceId: string, + memberId: string, + ): Promise; grantTreeAccess( spaceId: string, @@ -59,6 +85,14 @@ export interface CoreStore { principalId: string, treePath: string, ): Promise; + /** + * The raw grant rows in a space, optionally for a single principal. Distinct + * from buildTreeAccess, which resolves a member's *effective* access set. + */ + listTreeAccessGrants( + spaceId: string, + principalId?: string, + ): Promise; /** Resolve a member's effective grants in a space (for the space functions). */ buildTreeAccess(memberId: string, spaceId: string): Promise; @@ -73,6 +107,10 @@ export interface CoreStore { lookupId: string, secret: string, ): Promise; + getApiKey(id: string): Promise; + listApiKeys(memberId: string): Promise; + /** Hard-delete a key (revoke ≡ delete; there is no soft-revoke state). */ + deleteApiKey(id: string): Promise; /** Run operations atomically against the same transaction. */ withTransaction(fn: (db: CoreStore) => Promise): Promise; @@ -101,6 +139,39 @@ function mapPrincipal(row: Record): Principal { }; } +function mapSpaceMember(row: Record): SpaceMember { + return { + id: row.id as string, + kind: row.kind as PrincipalKind, + name: row.name as string, + ownerId: (row.owner_id as string | null) ?? null, + direct: Boolean(row.direct), + admin: Boolean(row.admin), + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapGroup(row: Record): Group { + return { + id: row.id as string, + name: row.name as string, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapApiKeyInfo(row: Record): ApiKeyInfo { + return { + id: row.id as string, + memberId: row.member_id as string, + lookupId: row.lookup_id as string, + name: row.name as string, + createdAt: row.created_at as Date, + expiresAt: (row.expires_at as Date | null) ?? null, + }; +} + export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { const sch = sql(schema); // escaped schema identifier reused across queries @@ -145,6 +216,36 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return row ? mapPrincipal(row) : null; }, + async getUserByName(name) { + const [row] = await sql`select * from ${sch}.get_user_by_name(${name})`; + return row ? mapPrincipal(row) : null; + }, + + async renamePrincipal(id, name) { + const [row] = await sql` + select ${sch}.rename_principal(${id}, ${name}) as ok + `; + return Boolean(row?.ok); + }, + + async deletePrincipal(id) { + const [row] = await sql`select ${sch}.delete_principal(${id}) as ok`; + return Boolean(row?.ok); + }, + + async listSpaceMembers(spaceId, kind) { + const rows = await sql` + select * from ${sch}.list_space_members(${spaceId}, ${kind ?? null}) + `; + return rows.map(mapSpaceMember); + }, + + async listSpaceGroups(spaceId) { + const rows = + await sql`select * from ${sch}.list_space_groups(${spaceId})`; + return rows.map(mapGroup); + }, + async addPrincipalToSpace(spaceId, principalId, admin = false) { await sql`select ${sch}.add_principal_to_space(${spaceId}, ${principalId}, ${admin})`; }, @@ -167,6 +268,35 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return Boolean(row?.removed); }, + async listGroupMembers(spaceId, groupId) { + const rows = await sql` + select * from ${sch}.list_group_members(${spaceId}, ${groupId}) + `; + return rows.map( + (r): GroupMember => ({ + memberId: r.member_id as string, + kind: r.kind as PrincipalKind, + name: r.name as string, + admin: Boolean(r.admin), + createdAt: r.created_at as Date, + }), + ); + }, + + async listGroupsForMember(spaceId, memberId) { + const rows = await sql` + select * from ${sch}.list_groups_for_member(${spaceId}, ${memberId}) + `; + return rows.map( + (r): GroupMembership => ({ + groupId: r.group_id as string, + name: r.name as string, + admin: Boolean(r.admin), + createdAt: r.created_at as Date, + }), + ); + }, + async grantTreeAccess(spaceId, principalId, treePath, access) { await sql` select ${sch}.grant_tree_access(${spaceId}, ${principalId}, ${treePath}::ltree, ${access}) @@ -180,6 +310,21 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return Boolean(row?.removed); }, + async listTreeAccessGrants(spaceId, principalId) { + const rows = await sql` + select * from ${sch}.list_tree_access_grants(${spaceId}, ${principalId ?? null}) + `; + return rows.map( + (r): TreeGrant => ({ + principalId: r.principal_id as string, + treePath: r.tree_path as string, + access: r.access as AccessLevel, + createdAt: r.created_at as Date, + updatedAt: (r.updated_at as Date | null) ?? null, + }), + ); + }, + async buildTreeAccess(memberId, spaceId) { const [row] = await sql` select ${sch}.build_tree_access(${memberId}, ${spaceId}) as ta @@ -213,6 +358,21 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { }; }, + async getApiKey(id) { + const [row] = await sql`select * from ${sch}.get_api_key(${id})`; + return row ? mapApiKeyInfo(row) : null; + }, + + async listApiKeys(memberId) { + const rows = await sql`select * from ${sch}.list_api_keys(${memberId})`; + return rows.map(mapApiKeyInfo); + }, + + async deleteApiKey(id) { + const [row] = await sql`select ${sch}.delete_api_key(${id}) as ok`; + return Boolean(row?.ok); + }, + async withTransaction(fn: (db: CoreStore) => Promise): Promise { return sql.begin((tx) => fn(coreStore(tx as unknown as Sql, schema)), diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts index 917c967..053d925 100644 --- a/packages/engine/core/index.ts +++ b/packages/engine/core/index.ts @@ -8,11 +8,17 @@ export { export { type CoreStore, coreStore } from "./db"; export type { AccessLevel, + ApiKeyInfo, CreatedApiKey, + Group, + GroupMember, + GroupMembership, Principal, PrincipalKind, Space, + SpaceMember, TreeAccess, + TreeGrant, ValidatedApiKey, } from "./types"; export { ACCESS, ROOT_PATH } from "./types"; diff --git a/packages/engine/core/types.ts b/packages/engine/core/types.ts index 1a36a65..e04df6c 100644 --- a/packages/engine/core/types.ts +++ b/packages/engine/core/types.ts @@ -68,3 +68,63 @@ export interface ValidatedApiKey { /** The api_key row id. */ apiKeyId: string; } + +/** + * A principal that belongs to a space — directly or through a group. + * `direct` is true for a direct (principal_space) membership; `admin` is the + * direct-membership admin flag (false for group-only members). + */ +export interface SpaceMember { + id: string; + kind: PrincipalKind; + name: string; + ownerId: string | null; + direct: boolean; + admin: boolean; + createdAt: Date; + updatedAt: Date | null; +} + +/** A group (kind 'g') belonging to a space. */ +export interface Group { + id: string; + name: string; + createdAt: Date; + updatedAt: Date | null; +} + +/** A member (user / agent) of a group, with the group admin flag. */ +export interface GroupMember { + memberId: string; + kind: PrincipalKind; + name: string; + admin: boolean; + createdAt: Date; +} + +/** A group a member belongs to, with the group admin flag. */ +export interface GroupMembership { + groupId: string; + name: string; + admin: boolean; + createdAt: Date; +} + +/** A tree-access grant row. */ +export interface TreeGrant { + principalId: string; + treePath: string; + access: AccessLevel; + createdAt: Date; + updatedAt: Date | null; +} + +/** Api key metadata (never includes the secret). */ +export interface ApiKeyInfo { + id: string; + memberId: string; + lookupId: string; + name: string; + createdAt: Date; + expiresAt: Date | null; +} From d5531fab5a78b8f15c355b97a36b0abce6f4154e Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 22:10:19 +0200 Subject: [PATCH 043/156] feat(server): space management + user RPC on the core model (4C-2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the legacy engine user/grant/owner/role/apiKey wire with a core-model management surface, split across two endpoints, plus the core primitives behind it. Endpoints - POST /api/v1/memory/rpc (space-scoped) gains: member.* (roster), group.* (groups), grant.* (3-level tree access), apiKey.* (agent keys). - POST /api/v1/user/rpc (new, session-only, user-scoped): agent.* lifecycle — agents are a user's global service accounts (per-owner-unique names), so they don't belong on a space endpoint. authenticateUser resolves the session user. Authority model - group.create/list/rename/delete: space admin (structural; owner@root is not enough). - group.addMember/removeMember/listMembers: space admin OR that group's admin (group_member.admin). group.listForMember: own memberships are self-service. - member.* and broad grant.list: space admin OR owner@root. - grant.set/remove and path-scoped grant.list: own-agent (capped) OR admin OR owner of the target tree path (subtree-owner delegation). - agent.*/apiKey.*: self-service over the agent you own; member.add lets a member bring their own agent into a space. Core - new SQL/coreStore: get_user_by_name, list_agents, list_space_members (Model 2: unions direct + via-group, flagged `direct`), list_space_groups, rename/ delete_principal, list_group_members, list_groups_for_member, list_tree_access _grants (optional principal + subtree filter), is_principal_space_admin / is_group_admin wrappers, get/list/delete_api_key. - fix member_groups: group membership is transitive (Model 2) — it no longer requires a principal_space row, and drops the broken join that required the group itself to be in principal_space (which silently disabled all group-inherited access). - shared rpc/core-error.ts maps core constraint violations to AppErrors. Protocol: new @memory.build/protocol/space and /user schema modules + method contracts. Integration tests cover both endpoints, the authority matrix, and group-inherited access. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../migrate/idempotent/002_group_member.sql | 41 +- .../core/migrate/idempotent/005_principal.sql | 26 + .../core/migrate/idempotent/007_grant.sql | 10 +- packages/engine/core/core.integration.test.ts | 13 + packages/engine/core/db.ts | 39 +- packages/protocol/package.json | 24 +- packages/protocol/space/api-key.ts | 59 +++ packages/protocol/space/grant.ts | 60 +++ packages/protocol/space/group.ts | 105 ++++ packages/protocol/space/index.ts | 118 +++++ packages/protocol/space/member.ts | 87 ++++ packages/protocol/user/agent.ts | 49 ++ packages/protocol/user/index.ts | 42 ++ .../server/middleware/authenticate-space.ts | 7 +- .../server/middleware/authenticate-user.ts | 48 ++ packages/server/router.ts | 19 + packages/server/rpc/core-error.ts | 32 ++ packages/server/rpc/index.ts | 6 + packages/server/rpc/memory/api-key.ts | 89 ++++ packages/server/rpc/memory/grant.ts | 113 ++++ packages/server/rpc/memory/group.ts | 182 +++++++ packages/server/rpc/memory/index.ts | 26 +- .../rpc/memory/management.integration.test.ts | 493 ++++++++++++++++++ packages/server/rpc/memory/member.ts | 103 ++++ .../rpc/memory/memory.integration.test.ts | 4 +- packages/server/rpc/memory/memory.ts | 2 +- packages/server/rpc/memory/support.ts | 259 +++++++++ packages/server/rpc/memory/types.ts | 2 + .../server/rpc/user/agent.integration.test.ts | 127 +++++ packages/server/rpc/user/agent.ts | 107 ++++ packages/server/rpc/user/index.ts | 13 + packages/server/rpc/user/types.ts | 31 ++ 32 files changed, 2312 insertions(+), 24 deletions(-) create mode 100644 packages/protocol/space/api-key.ts create mode 100644 packages/protocol/space/grant.ts create mode 100644 packages/protocol/space/group.ts create mode 100644 packages/protocol/space/index.ts create mode 100644 packages/protocol/space/member.ts create mode 100644 packages/protocol/user/agent.ts create mode 100644 packages/protocol/user/index.ts create mode 100644 packages/server/middleware/authenticate-user.ts create mode 100644 packages/server/rpc/core-error.ts create mode 100644 packages/server/rpc/memory/api-key.ts create mode 100644 packages/server/rpc/memory/grant.ts create mode 100644 packages/server/rpc/memory/group.ts create mode 100644 packages/server/rpc/memory/management.integration.test.ts create mode 100644 packages/server/rpc/memory/member.ts create mode 100644 packages/server/rpc/memory/support.ts create mode 100644 packages/server/rpc/user/agent.integration.test.ts create mode 100644 packages/server/rpc/user/agent.ts create mode 100644 packages/server/rpc/user/index.ts create mode 100644 packages/server/rpc/user/types.ts diff --git a/packages/database/core/migrate/idempotent/002_group_member.sql b/packages/database/core/migrate/idempotent/002_group_member.sql index b2c7a8b..f5a3fdd 100644 --- a/packages/database/core/migrate/idempotent/002_group_member.sql +++ b/packages/database/core/migrate/idempotent/002_group_member.sql @@ -11,16 +11,39 @@ returns table , admin bool ) as $func$ + -- Group membership is space-scoped by group_member.space_id and confers + -- space access transitively — a member need NOT have a direct principal_space + -- row to inherit a group's grants (Model 2). The FKs already constrain + -- group_id to a group and member_id to a user/agent. select gm.group_id - , gm.admin and (not m.kind = 'a') -- agent's cannot be group admins - from {{schema}}.principal m -- the member - -- assert the member belongs to the space - inner join {{schema}}.principal_space psm on (m.id = psm.principal_id and psm.space_id = _space_id) - -- find the groups the member belongs to in the space - inner join {{schema}}.group_member gm on (m.member_id = gm.member_id and gm.space_id = _space_id) - -- assert the group belongs to the space - inner join {{schema}}.principal_space psg on (gm.group_id = psg.principal_id and psg.space_id = _space_id) - where m.member_id = _member_id -- the member + , gm.admin and (not m.kind = 'a') -- agents cannot be group admins + from {{schema}}.group_member gm + inner join {{schema}}.principal m on (m.member_id = gm.member_id) + where gm.member_id = _member_id + and gm.space_id = _space_id $func$ language sql stable security invoker ; + +------------------------------------------------------------------------------- +-- is_group_admin +-- Whether a member is an admin of a specific group in a space. (Agents are +-- never group admins — enforced by member_groups.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.is_group_admin +( _member_id uuid +, _group_id uuid +, _space_id uuid +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.member_groups(_member_id, _space_id) mg + where mg.group_id = _group_id + and mg.admin + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/005_principal.sql b/packages/database/core/migrate/idempotent/005_principal.sql index e27ef4f..0bf6dbf 100644 --- a/packages/database/core/migrate/idempotent/005_principal.sql +++ b/packages/database/core/migrate/idempotent/005_principal.sql @@ -101,6 +101,32 @@ $func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; +------------------------------------------------------------------------------- +-- list_agents +-- A user's agents (global; agents are owned by a user, not scoped to a space). +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_agents +( _owner_id uuid +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, space_id uuid +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.kind, p.name::text, p.owner_id, p.space_id, p.created_at, p.updated_at + from {{schema}}.principal p + where p.kind = 'a' + and p.owner_id = _owner_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + ------------------------------------------------------------------------------- -- list_space_groups -- All groups belonging to a space (groups are space-scoped via space_id). diff --git a/packages/database/core/migrate/idempotent/007_grant.sql b/packages/database/core/migrate/idempotent/007_grant.sql index 25ea58b..6dbf5a0 100644 --- a/packages/database/core/migrate/idempotent/007_grant.sql +++ b/packages/database/core/migrate/idempotent/007_grant.sql @@ -47,13 +47,16 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ------------------------------------------------------------------------------- -- list_tree_access_grants --- The grant rows in a space, optionally for a single principal. (Owner listing --- is this filtered to access = 3 by the caller.) Distinct from build_tree_access, --- which resolves a member's *effective* access set; this lists the raw grants. +-- The grant rows in a space, optionally filtered to a single principal and/or +-- to a subtree (_under: only grants at-or-below this path, i.e. _under @> path). +-- (Owner listing is this filtered to access = 3 by the caller.) Distinct from +-- build_tree_access, which resolves a member's *effective* access set; this +-- lists the raw grants. ------------------------------------------------------------------------------- create or replace function {{schema}}.list_tree_access_grants ( _space_id uuid , _principal_id uuid default null +, _under ltree default null ) returns table ( principal_id uuid @@ -67,6 +70,7 @@ as $func$ from {{schema}}.tree_access t where t.space_id = _space_id and (_principal_id is null or t.principal_id = _principal_id) + and (_under is null or _under @> t.tree_path) order by t.principal_id, t.tree_path $func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts index 2709d88..7dc6e95 100644 --- a/packages/engine/core/core.integration.test.ts +++ b/packages/engine/core/core.integration.test.ts @@ -130,6 +130,19 @@ test("groups: create, list, rename, members, delete", async () => { expect(await core.listSpaceGroups(spaceId)).toHaveLength(0); }); +test("group grants are inherited transitively (Model 2)", async () => { + // a user who is ONLY a group member (no direct principal_space row, no direct + // grant) inherits the group's grant via build_tree_access. + const groupOnly = await v7(); + await core.createUser(groupOnly, `go_${rand(8)}@example.com`); + const groupId = await core.createGroup(spaceId, `grp_${rand(6)}`); + await core.addGroupMember(spaceId, groupId, groupOnly); + await core.grantTreeAccess(spaceId, groupId, "shared", ACCESS.write); + + const ta = await core.buildTreeAccess(groupOnly, spaceId); + expect(ta).toContainEqual({ tree_path: "shared", access: ACCESS.write }); +}); + test("listTreeAccessGrants returns grants; filterable by principal", async () => { await core.grantTreeAccess(spaceId, userId, "a.b", ACCESS.write); await core.grantTreeAccess(spaceId, userId, "c", ACCESS.owner); diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index 6ab4831..234885f 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -43,8 +43,18 @@ export interface CoreStore { spaceId: string, kind?: PrincipalKind, ): Promise; + /** Whether a principal is an admin of a space (agents are never admins). */ + isSpaceAdmin(principalId: string, spaceId: string): Promise; + /** Whether a member is an admin of a group (agents are never group admins). */ + isGroupAdmin( + memberId: string, + groupId: string, + spaceId: string, + ): Promise; /** Groups belonging to a space. */ listSpaceGroups(spaceId: string): Promise; + /** A user's agents (global; agents are owned by a user, not a space). */ + listAgents(ownerId: string): Promise; addPrincipalToSpace( spaceId: string, @@ -86,12 +96,14 @@ export interface CoreStore { treePath: string, ): Promise; /** - * The raw grant rows in a space, optionally for a single principal. Distinct + * The raw grant rows in a space, optionally for a single principal and/or + * restricted to a subtree (`under`: grants at-or-below this path). Distinct * from buildTreeAccess, which resolves a member's *effective* access set. */ listTreeAccessGrants( spaceId: string, principalId?: string, + under?: string, ): Promise; /** Resolve a member's effective grants in a space (for the space functions). */ @@ -246,6 +258,25 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return rows.map(mapGroup); }, + async listAgents(ownerId) { + const rows = await sql`select * from ${sch}.list_agents(${ownerId})`; + return rows.map(mapPrincipal); + }, + + async isSpaceAdmin(principalId, spaceId) { + const [row] = await sql` + select ${sch}.is_principal_space_admin(${principalId}, ${spaceId}) as ok + `; + return Boolean(row?.ok); + }, + + async isGroupAdmin(memberId, groupId, spaceId) { + const [row] = await sql` + select ${sch}.is_group_admin(${memberId}, ${groupId}, ${spaceId}) as ok + `; + return Boolean(row?.ok); + }, + async addPrincipalToSpace(spaceId, principalId, admin = false) { await sql`select ${sch}.add_principal_to_space(${spaceId}, ${principalId}, ${admin})`; }, @@ -310,9 +341,11 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return Boolean(row?.removed); }, - async listTreeAccessGrants(spaceId, principalId) { + async listTreeAccessGrants(spaceId, principalId, under) { const rows = await sql` - select * from ${sch}.list_tree_access_grants(${spaceId}, ${principalId ?? null}) + select * from ${sch}.list_tree_access_grants( + ${spaceId}, ${principalId ?? null}, ${under ?? null}::ltree + ) `; return rows.map( (r): TreeGrant => ({ diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 52b3670..091a23f 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -44,6 +44,26 @@ "types": "./engine/*.ts", "import": "./dist/engine/*.js" }, + "./space": { + "bun": "./space/index.ts", + "types": "./space/index.ts", + "import": "./dist/space/index.js" + }, + "./space/*": { + "bun": "./space/*.ts", + "types": "./space/*.ts", + "import": "./dist/space/*.js" + }, + "./user": { + "bun": "./user/index.ts", + "types": "./user/index.ts", + "import": "./dist/user/index.js" + }, + "./user/*": { + "bun": "./user/*.ts", + "types": "./user/*.ts", + "import": "./dist/user/*.js" + }, "./accounts": { "bun": "./accounts/index.ts", "types": "./accounts/index.ts", @@ -64,12 +84,14 @@ "dist", "*.ts", "engine/*.ts", + "space/*.ts", + "user/*.ts", "accounts/*.ts", "auth/*.ts", "!*.test.ts" ], "scripts": { - "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", + "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/member.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", "build:types": "tsc -p tsconfig.build.json", "build": "../../bun run build:js && ../../bun run build:types", "prepublishOnly": "../../bun run build" diff --git a/packages/protocol/space/api-key.ts b/packages/protocol/space/api-key.ts new file mode 100644 index 0000000..faaa372 --- /dev/null +++ b/packages/protocol/space/api-key.ts @@ -0,0 +1,59 @@ +/** + * Api key method schemas (apiKey.*). + * + * Keys are agent-only (humans authenticate via session). The plaintext key is + * returned exactly once, by apiKey.create. There is no soft-revoke state: + * apiKey.delete is the only removal (revoke ≡ delete). + */ +import { z } from "zod"; +import { nameSchema, timestampSchema, uuidv7Schema } from "../fields.ts"; + +export const apiKeyInfoResponse = z.object({ + id: z.string(), + memberId: z.string(), + lookupId: z.string(), + name: z.string(), + createdAt: z.string(), + expiresAt: z.string().nullable(), +}); +export type ApiKeyInfoResponse = z.infer; + +// apiKey.create — mint a key for an agent in this space +export const apiKeyCreateParams = z.object({ + agentId: uuidv7Schema, + name: nameSchema, + expiresAt: timestampSchema.optional().nullable(), +}); +export type ApiKeyCreateParams = z.infer; + +export const apiKeyCreateResult = z.object({ + id: z.string(), + /** The full api key string — returned once; only its hash is stored. */ + key: z.string(), +}); +export type ApiKeyCreateResult = z.infer; + +// apiKey.list — a member's keys (metadata only) +export const apiKeyListParams = z.object({ memberId: uuidv7Schema }); +export type ApiKeyListParams = z.infer; + +export const apiKeyListResult = z.object({ + apiKeys: z.array(apiKeyInfoResponse), +}); +export type ApiKeyListResult = z.infer; + +// apiKey.get +export const apiKeyGetParams = z.object({ id: uuidv7Schema }); +export type ApiKeyGetParams = z.infer; + +export const apiKeyGetResult = z.object({ + apiKey: apiKeyInfoResponse.nullable(), +}); +export type ApiKeyGetResult = z.infer; + +// apiKey.delete (revoke ≡ delete) +export const apiKeyDeleteParams = z.object({ id: uuidv7Schema }); +export type ApiKeyDeleteParams = z.infer; + +export const apiKeyDeleteResult = z.object({ deleted: z.boolean() }); +export type ApiKeyDeleteResult = z.infer; diff --git a/packages/protocol/space/grant.ts b/packages/protocol/space/grant.ts new file mode 100644 index 0000000..bec967a --- /dev/null +++ b/packages/protocol/space/grant.ts @@ -0,0 +1,60 @@ +/** + * Tree-access grant method schemas (grant.*). + * + * The core model uses three additive levels (1 = read, 2 = write, 3 = owner); + * there are no per-action grants and no deny entries. Owner listing is grant.list + * filtered to access = 3. + */ +import { z } from "zod"; +import { treePathSchema, uuidv7Schema } from "../fields.ts"; + +/** Access level: 1 = read, 2 = write, 3 = owner. */ +export const accessLevelSchema = z.union([ + z.literal(1), + z.literal(2), + z.literal(3), +]); +export type AccessLevel = z.infer; + +export const treeGrantResponse = z.object({ + principalId: z.string(), + treePath: z.string(), + access: accessLevelSchema, + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type TreeGrantResponse = z.infer; + +// grant.set — grant or update a principal's access at a tree path +export const grantSetParams = z.object({ + principalId: uuidv7Schema, + treePath: treePathSchema, + access: accessLevelSchema, +}); +export type GrantSetParams = z.infer; + +export const grantSetResult = z.object({ granted: z.boolean() }); +export type GrantSetResult = z.infer; + +// grant.remove +export const grantRemoveParams = z.object({ + principalId: uuidv7Schema, + treePath: treePathSchema, +}); +export type GrantRemoveParams = z.infer; + +export const grantRemoveResult = z.object({ removed: z.boolean() }); +export type GrantRemoveResult = z.infer; + +// grant.list — optionally filtered to a principal and/or a subtree path +export const grantListParams = z.object({ + principalId: uuidv7Schema.optional().nullable(), + /** Only grants at or below this tree path (requires owning the path). */ + treePath: treePathSchema.optional().nullable(), +}); +export type GrantListParams = z.infer; + +export const grantListResult = z.object({ + grants: z.array(treeGrantResponse), +}); +export type GrantListResult = z.infer; diff --git a/packages/protocol/space/group.ts b/packages/protocol/space/group.ts new file mode 100644 index 0000000..4b686e6 --- /dev/null +++ b/packages/protocol/space/group.ts @@ -0,0 +1,105 @@ +/** + * Group method schemas (group.*). + * + * Groups are space-scoped principals used to bundle members for tree-access + * grants. Group membership confers space access (a group member is a space + * member, flagged direct=false in member.list). + */ +import { z } from "zod"; +import { nameSchema, uuidv7Schema } from "../fields.ts"; +import { principalKindSchema } from "./member.ts"; + +export const groupResponse = z.object({ + id: z.string(), + name: z.string(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type GroupResponse = z.infer; + +export const groupMemberResponse = z.object({ + memberId: z.string(), + kind: principalKindSchema, + name: z.string(), + admin: z.boolean(), + createdAt: z.string(), +}); +export type GroupMemberResponse = z.infer; + +export const groupMembershipResponse = z.object({ + groupId: z.string(), + name: z.string(), + admin: z.boolean(), + createdAt: z.string(), +}); +export type GroupMembershipResponse = z.infer; + +// group.create +export const groupCreateParams = z.object({ name: nameSchema }); +export type GroupCreateParams = z.infer; + +export const groupCreateResult = z.object({ id: z.string() }); +export type GroupCreateResult = z.infer; + +// group.list +export const groupListParams = z.object({}); +export type GroupListParams = z.infer; + +export const groupListResult = z.object({ groups: z.array(groupResponse) }); +export type GroupListResult = z.infer; + +// group.rename +export const groupRenameParams = z.object({ + id: uuidv7Schema, + name: nameSchema, +}); +export type GroupRenameParams = z.infer; + +export const groupRenameResult = z.object({ renamed: z.boolean() }); +export type GroupRenameResult = z.infer; + +// group.delete +export const groupDeleteParams = z.object({ id: uuidv7Schema }); +export type GroupDeleteParams = z.infer; + +export const groupDeleteResult = z.object({ deleted: z.boolean() }); +export type GroupDeleteResult = z.infer; + +// group.addMember +export const groupAddMemberParams = z.object({ + groupId: uuidv7Schema, + memberId: uuidv7Schema, + admin: z.boolean().optional(), +}); +export type GroupAddMemberParams = z.infer; + +export const groupAddMemberResult = z.object({ added: z.boolean() }); +export type GroupAddMemberResult = z.infer; + +// group.removeMember +export const groupRemoveMemberParams = z.object({ + groupId: uuidv7Schema, + memberId: uuidv7Schema, +}); +export type GroupRemoveMemberParams = z.infer; + +export const groupRemoveMemberResult = z.object({ removed: z.boolean() }); +export type GroupRemoveMemberResult = z.infer; + +// group.listMembers +export const groupListMembersParams = z.object({ groupId: uuidv7Schema }); +export type GroupListMembersParams = z.infer; + +export const groupListMembersResult = z.object({ + members: z.array(groupMemberResponse), +}); +export type GroupListMembersResult = z.infer; + +// group.listForMember +export const groupListForMemberParams = z.object({ memberId: uuidv7Schema }); +export type GroupListForMemberParams = z.infer; + +export const groupListForMemberResult = z.object({ + groups: z.array(groupMembershipResponse), +}); +export type GroupListForMemberResult = z.infer; diff --git a/packages/protocol/space/index.ts b/packages/protocol/space/index.ts new file mode 100644 index 0000000..a2c54e3 --- /dev/null +++ b/packages/protocol/space/index.ts @@ -0,0 +1,118 @@ +/** + * Space management RPC contract — the control-plane methods served on + * POST /api/v1/memory/rpc alongside the memory.* data-plane methods. + * + * Follows the core model: principals (users/agents/groups), space membership, + * group membership, 3-level tree-access grants, and agent api keys. (Agent + * lifecycle is user-scoped and lives on the user endpoint; here agents are only + * referenced as members / api-key holders.) + */ +import type { z } from "zod"; +import { + apiKeyCreateParams, + apiKeyCreateResult, + apiKeyDeleteParams, + apiKeyDeleteResult, + apiKeyGetParams, + apiKeyGetResult, + apiKeyListParams, + apiKeyListResult, +} from "./api-key.ts"; +import { + grantListParams, + grantListResult, + grantRemoveParams, + grantRemoveResult, + grantSetParams, + grantSetResult, +} from "./grant.ts"; +import { + groupAddMemberParams, + groupAddMemberResult, + groupCreateParams, + groupCreateResult, + groupDeleteParams, + groupDeleteResult, + groupListForMemberParams, + groupListForMemberResult, + groupListMembersParams, + groupListMembersResult, + groupListParams, + groupListResult, + groupRemoveMemberParams, + groupRemoveMemberResult, + groupRenameParams, + groupRenameResult, +} from "./group.ts"; +import { + memberAddParams, + memberAddResult, + memberListParams, + memberListResult, + memberRemoveParams, + memberRemoveResult, + memberResolveByEmailParams, + memberResolveByEmailResult, +} from "./member.ts"; + +export * from "./api-key.ts"; +export * from "./grant.ts"; +export * from "./group.ts"; +export * from "./member.ts"; + +function method( + params: TParams, + result: TResult, +) { + return { params, result }; +} + +/** + * Space management RPC method contract (member/agent/group/grant/apiKey). + * Served on the memory endpoint together with the memory.* methods. + */ +export const spaceMethods = { + // Membership (4) + "member.list": method(memberListParams, memberListResult), + "member.add": method(memberAddParams, memberAddResult), + "member.remove": method(memberRemoveParams, memberRemoveResult), + "member.resolveByEmail": method( + memberResolveByEmailParams, + memberResolveByEmailResult, + ), + + // Groups (8) + "group.create": method(groupCreateParams, groupCreateResult), + "group.list": method(groupListParams, groupListResult), + "group.rename": method(groupRenameParams, groupRenameResult), + "group.delete": method(groupDeleteParams, groupDeleteResult), + "group.addMember": method(groupAddMemberParams, groupAddMemberResult), + "group.removeMember": method( + groupRemoveMemberParams, + groupRemoveMemberResult, + ), + "group.listMembers": method(groupListMembersParams, groupListMembersResult), + "group.listForMember": method( + groupListForMemberParams, + groupListForMemberResult, + ), + + // Grants (3) + "grant.set": method(grantSetParams, grantSetResult), + "grant.remove": method(grantRemoveParams, grantRemoveResult), + "grant.list": method(grantListParams, grantListResult), + + // Api keys (4) + "apiKey.create": method(apiKeyCreateParams, apiKeyCreateResult), + "apiKey.list": method(apiKeyListParams, apiKeyListResult), + "apiKey.get": method(apiKeyGetParams, apiKeyGetResult), + "apiKey.delete": method(apiKeyDeleteParams, apiKeyDeleteResult), +} as const; + +export type SpaceMethodName = keyof typeof spaceMethods; +export type SpaceParams = z.infer< + (typeof spaceMethods)[M]["params"] +>; +export type SpaceResult = z.infer< + (typeof spaceMethods)[M]["result"] +>; diff --git a/packages/protocol/space/member.ts b/packages/protocol/space/member.ts new file mode 100644 index 0000000..a2e17b9 --- /dev/null +++ b/packages/protocol/space/member.ts @@ -0,0 +1,87 @@ +/** + * Space membership method schemas (member.*). + * + * The space management API, served on POST /api/v1/memory/rpc, follows the core + * model: principals (users/agents/groups), space membership, group membership, + * 3-level tree-access grants, and agent api keys. All methods are scoped to the + * space selected by the X-Me-Space header and require space-owner authority. + */ +import { z } from "zod"; +import { emailSchema, nameSchema, uuidv7Schema } from "../fields.ts"; + +/** Principal kind: user / group / agent. */ +export const principalKindSchema = z.enum(["u", "g", "a"]); +export type PrincipalKind = z.infer; + +/** + * A principal that belongs to a space — directly or via a group. + * `direct` is true for a direct membership; `admin` is the direct-membership + * admin flag (false for group-only members). + */ +export const spaceMemberResponse = z.object({ + id: z.string(), + kind: principalKindSchema, + name: z.string(), + ownerId: z.string().nullable(), + direct: z.boolean(), + admin: z.boolean(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type SpaceMemberResponse = z.infer; + +/** A resolved principal (used by member.resolveByEmail). */ +export const principalResponse = z.object({ + id: z.string(), + kind: principalKindSchema, + name: z.string(), + ownerId: z.string().nullable(), + spaceId: z.string().nullable(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type PrincipalResponse = z.infer; + +// member.list +export const memberListParams = z.object({ + kind: principalKindSchema.optional().nullable(), +}); +export type MemberListParams = z.infer; + +export const memberListResult = z.object({ + members: z.array(spaceMemberResponse), +}); +export type MemberListResult = z.infer; + +// member.add +export const memberAddParams = z.object({ + principalId: uuidv7Schema, + admin: z.boolean().optional(), +}); +export type MemberAddParams = z.infer; + +export const memberAddResult = z.object({ added: z.boolean() }); +export type MemberAddResult = z.infer; + +// member.remove +export const memberRemoveParams = z.object({ principalId: uuidv7Schema }); +export type MemberRemoveParams = z.infer; + +export const memberRemoveResult = z.object({ removed: z.boolean() }); +export type MemberRemoveResult = z.infer; + +// member.resolveByEmail — find a global user by email (to add them to the space) +export const memberResolveByEmailParams = z.object({ email: emailSchema }); +export type MemberResolveByEmailParams = z.infer< + typeof memberResolveByEmailParams +>; + +export const memberResolveByEmailResult = z.object({ + principal: principalResponse.nullable(), +}); +export type MemberResolveByEmailResult = z.infer< + typeof memberResolveByEmailResult +>; + +// shared by agent.* / group.* mutation results +export { nameSchema }; diff --git a/packages/protocol/user/agent.ts b/packages/protocol/user/agent.ts new file mode 100644 index 0000000..90e3bfe --- /dev/null +++ b/packages/protocol/user/agent.ts @@ -0,0 +1,49 @@ +/** + * Agent method schemas (agent.*) for the user RPC. + * + * Agents are a user's global service accounts (owned by the user, names unique + * per user, not scoped to a space). Their lifecycle lives on the session-only + * user endpoint (POST /api/v1/user/rpc); bringing an agent into a space and + * minting its (space-bound) api key are space-endpoint operations. + */ +import { z } from "zod"; +import { nameSchema, uuidv7Schema } from "../fields.ts"; + +export const agentResponse = z.object({ + id: z.string(), + name: z.string(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type AgentResponse = z.infer; + +// agent.create — create an agent owned by the calling user +export const agentCreateParams = z.object({ name: nameSchema }); +export type AgentCreateParams = z.infer; + +export const agentCreateResult = z.object({ id: z.string() }); +export type AgentCreateResult = z.infer; + +// agent.list — the caller's agents +export const agentListParams = z.object({}); +export type AgentListParams = z.infer; + +export const agentListResult = z.object({ agents: z.array(agentResponse) }); +export type AgentListResult = z.infer; + +// agent.rename +export const agentRenameParams = z.object({ + id: uuidv7Schema, + name: nameSchema, +}); +export type AgentRenameParams = z.infer; + +export const agentRenameResult = z.object({ renamed: z.boolean() }); +export type AgentRenameResult = z.infer; + +// agent.delete +export const agentDeleteParams = z.object({ id: uuidv7Schema }); +export type AgentDeleteParams = z.infer; + +export const agentDeleteResult = z.object({ deleted: z.boolean() }); +export type AgentDeleteResult = z.infer; diff --git a/packages/protocol/user/index.ts b/packages/protocol/user/index.ts new file mode 100644 index 0000000..4e6f9fa --- /dev/null +++ b/packages/protocol/user/index.ts @@ -0,0 +1,42 @@ +/** + * User RPC contract — session-only, user-scoped methods served on + * POST /api/v1/user/rpc. Covers the lifecycle of a user's global service + * accounts (agents); space membership and api keys live on the space endpoint. + */ +import type { z } from "zod"; + +import { + agentCreateParams, + agentCreateResult, + agentDeleteParams, + agentDeleteResult, + agentListParams, + agentListResult, + agentRenameParams, + agentRenameResult, +} from "./agent.ts"; + +export * from "./agent.ts"; + +function method( + params: TParams, + result: TResult, +) { + return { params, result }; +} + +/** User RPC method contract (agent lifecycle). */ +export const userMethods = { + "agent.create": method(agentCreateParams, agentCreateResult), + "agent.list": method(agentListParams, agentListResult), + "agent.rename": method(agentRenameParams, agentRenameResult), + "agent.delete": method(agentDeleteParams, agentDeleteResult), +} as const; + +export type UserMethodName = keyof typeof userMethods; +export type UserParams = z.infer< + (typeof userMethods)[M]["params"] +>; +export type UserResult = z.infer< + (typeof userMethods)[M]["result"] +>; diff --git a/packages/server/middleware/authenticate-space.ts b/packages/server/middleware/authenticate-space.ts index 2640116..a47d1c5 100644 --- a/packages/server/middleware/authenticate-space.ts +++ b/packages/server/middleware/authenticate-space.ts @@ -47,6 +47,8 @@ export interface SpaceAuthContext { apiKeyId: string | null; /** The principal's effective grants in this space — the access gate. */ treeAccess: TreeAccess; + /** Whether the principal is a space admin (principal_space.admin). */ + admin: boolean; } export type SpaceAuthResult = @@ -155,8 +157,10 @@ async function authenticateSpaceInner( return { ok: false, error: forbidden("No access to this space") }; } - // 6. Bind the data-plane store to this space's schema. + // 6. Bind the data-plane store to this space's schema, and resolve whether + // the principal is a space admin (membership-level management authority). const store = spaceStore(db, slugToSchema(space.slug)); + const admin = await core.isSpaceAdmin(principalId, space.id); debug("space auth succeeded", { slug, @@ -173,6 +177,7 @@ async function authenticateSpaceInner( principalId, apiKeyId, treeAccess, + admin, }, }; } diff --git a/packages/server/middleware/authenticate-user.ts b/packages/server/middleware/authenticate-user.ts new file mode 100644 index 0000000..ac665d9 --- /dev/null +++ b/packages/server/middleware/authenticate-user.ts @@ -0,0 +1,48 @@ +/** + * Authentication for the user RPC (`/api/v1/user/rpc`). + * + * User-scoped, session-only: it resolves the calling human (a user principal) + * from a session token. Api keys are agent credentials and do not authenticate + * here (agents can't manage agents), so an api-key token simply fails session + * validation → 401. + */ +import type { AuthStore } from "@memory.build/auth"; +import { debug, span } from "@pydantic/logfire-node"; +import { unauthorized } from "../util/response"; +import { extractBearerToken } from "./authenticate"; + +export interface UserAuthContext { + type: "user"; + /** The authenticated user id (== the core user-principal id). */ + userId: string; +} + +export type UserAuthResult = + | { ok: true; context: UserAuthContext } + | { ok: false; error: Response }; + +export async function authenticateUser( + request: Request, + auth: AuthStore, +): Promise { + return span("auth.user", { + attributes: { "auth.type": "user" }, + callback: async () => { + const token = extractBearerToken(request); + if (!token) { + debug("user auth failed: missing Authorization header"); + return { + ok: false, + error: unauthorized("Missing or invalid Authorization header"), + }; + } + const session = await auth.validateSession(token); + if (!session) { + debug("user auth failed: invalid or expired session"); + return { ok: false, error: unauthorized("Invalid or expired session") }; + } + debug("user auth succeeded", { userId: session.userId }); + return { ok: true, context: { type: "user", userId: session.userId } }; + }, + }); +} diff --git a/packages/server/router.ts b/packages/server/router.ts index 7cfa722..1792bce 100644 --- a/packages/server/router.ts +++ b/packages/server/router.ts @@ -15,12 +15,14 @@ import { authenticateEngine, } from "./middleware/authenticate"; import { authenticateSpace } from "./middleware/authenticate-space"; +import { authenticateUser } from "./middleware/authenticate-user"; import { checkClientVersion } from "./middleware/client-version"; import { accountsMethods, createRpcHandler, engineMethods, memoryMethods, + userMethods, } from "./rpc"; import { notFound } from "./util/response"; @@ -223,10 +225,20 @@ export function createRouter(ctx: ServerContext): Router { principalId: spaceContext.principalId, apiKeyId: spaceContext.apiKeyId, treeAccess: spaceContext.treeAccess, + admin: spaceContext.admin, embeddingConfig, }; }); + // User RPC (new model): session-only, user-scoped (agent lifecycle) + const userRpcHandler = createRpcHandler(userMethods, async (request) => { + const result = await authenticateUser(request, auth); + if (!result.ok) { + return result.error; + } + return { core, userId: result.context.userId }; + }); + /** * Application routes. * @@ -312,6 +324,13 @@ export function createRouter(ctx: ServerContext): Router { pattern: "/api/v1/memory/rpc", handler: withClientVersionCheck(memoryRpcHandler), }, + + // User RPC (new model: session-only, user-scoped agent lifecycle) + { + method: "POST", + pattern: "/api/v1/user/rpc", + handler: withClientVersionCheck(userRpcHandler), + }, ]; /** diff --git a/packages/server/rpc/core-error.ts b/packages/server/rpc/core-error.ts new file mode 100644 index 0000000..3cf3150 --- /dev/null +++ b/packages/server/rpc/core-error.ts @@ -0,0 +1,32 @@ +/** + * Map core control-plane SQL constraint violations to AppErrors, shared by the + * space management and user RPC handlers. + */ +import { AppError } from "./errors"; + +/** + * Unique → CONFLICT (e.g. duplicate name); foreign-key / check / bad-input → + * VALIDATION_ERROR; everything else propagates. + */ +function mapCoreError(e: unknown): never { + const code = (e as { code?: string }).code; + if (code === "23505") { + throw new AppError("CONFLICT", "A record with that name already exists"); + } + if (code === "23503" || code === "23514" || code === "22P02") { + throw new AppError( + "VALIDATION_ERROR", + e instanceof Error ? e.message : "Invalid parameter", + ); + } + throw e instanceof Error ? e : new Error(String(e)); +} + +/** Run a coreStore call, mapping constraint violations to AppErrors. */ +export async function guardCore(fn: () => Promise): Promise { + try { + return await fn(); + } catch (e) { + return mapCoreError(e); + } +} diff --git a/packages/server/rpc/index.ts b/packages/server/rpc/index.ts index 525f00a..c2fc516 100644 --- a/packages/server/rpc/index.ts +++ b/packages/server/rpc/index.ts @@ -52,3 +52,9 @@ export type { MethodRegistry, RegisteredMethod, } from "./types"; +export { + assertUserRpcContext, + isUserRpcContext, + type UserRpcContext, + userMethods, +} from "./user"; diff --git a/packages/server/rpc/memory/api-key.ts b/packages/server/rpc/memory/api-key.ts new file mode 100644 index 0000000..af93e31 --- /dev/null +++ b/packages/server/rpc/memory/api-key.ts @@ -0,0 +1,89 @@ +/** + * Api key handlers (apiKey.*). Keys are agent-only and self-service: the caller + * manages keys for agents they own. The plaintext key is returned once by + * create. Revoke ≡ delete (no soft-revoke state). + */ +import { formatApiKey } from "@memory.build/engine/core"; +import type { + ApiKeyCreateParams, + ApiKeyCreateResult, + ApiKeyDeleteParams, + ApiKeyDeleteResult, + ApiKeyGetParams, + ApiKeyGetResult, + ApiKeyListParams, + ApiKeyListResult, +} from "@memory.build/protocol/space"; +import { + apiKeyCreateParams, + apiKeyDeleteParams, + apiKeyGetParams, + apiKeyListParams, +} from "@memory.build/protocol/space"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { guardCore, requireOwnedAgent, toApiKeyInfoResponse } from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +async function apiKeyCreate( + params: ApiKeyCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Keys are agent-only; the caller must own the agent (which is in this space). + await requireOwnedAgent(ctx, params.agentId); + + const created = await guardCore(() => + ctx.core.createApiKey(params.agentId, params.name, { + expiresAt: params.expiresAt ? new Date(params.expiresAt) : undefined, + }), + ); + // The full key string embeds the space slug for routing; returned once. + const key = formatApiKey(ctx.space.slug, created.lookupId, created.secret); + return { id: created.id, key }; +} + +async function apiKeyList( + params: ApiKeyListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireOwnedAgent(ctx, params.memberId); + const keys = await ctx.core.listApiKeys(params.memberId); + return { apiKeys: keys.map(toApiKeyInfoResponse) }; +} + +async function apiKeyGet( + params: ApiKeyGetParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const key = await ctx.core.getApiKey(params.id); + if (!key) return { apiKey: null }; + // Only the owning user of the key's agent may see it. + await requireOwnedAgent(ctx, key.memberId); + return { apiKey: toApiKeyInfoResponse(key) }; +} + +async function apiKeyDelete( + params: ApiKeyDeleteParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const key = await ctx.core.getApiKey(params.id); + if (!key) return { deleted: false }; + await requireOwnedAgent(ctx, key.memberId); + const deleted = await guardCore(() => ctx.core.deleteApiKey(params.id)); + return { deleted }; +} + +export const apiKeyMethods = buildRegistry() + .register("apiKey.create", apiKeyCreateParams, apiKeyCreate) + .register("apiKey.list", apiKeyListParams, apiKeyList) + .register("apiKey.get", apiKeyGetParams, apiKeyGet) + .register("apiKey.delete", apiKeyDeleteParams, apiKeyDelete) + .build(); diff --git a/packages/server/rpc/memory/grant.ts b/packages/server/rpc/memory/grant.ts new file mode 100644 index 0000000..c044363 --- /dev/null +++ b/packages/server/rpc/memory/grant.ts @@ -0,0 +1,113 @@ +/** + * Tree-access grant handlers (grant.*). Three additive levels + * (1 = read, 2 = write, 3 = owner); owner listing is grant.list filtered to 3. + */ +import type { + GrantListParams, + GrantListResult, + GrantRemoveParams, + GrantRemoveResult, + GrantSetParams, + GrantSetResult, +} from "@memory.build/protocol/space"; +import { + grantListParams, + grantRemoveParams, + grantSetParams, +} from "@memory.build/protocol/space"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + callerOwnsAgent, + guardCore, + isSpaceManager, + ownsTreePath, + requireSpaceManager, + requireTreeOwner, + toTreeGrantResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +/** + * Authority to grant/remove access at a path. Allowed when any of: + * - the target is the caller's OWN agent (self-service — capped anyway); + * - the caller is a space admin / owner; + * - the caller owns the tree path (owning a subtree delegates control within it). + */ +async function requireGrantAuthority( + ctx: SpaceRpcContext, + principalId: string, + treePath: string, +): Promise { + if (await callerOwnsAgent(ctx, principalId)) return; + if (isSpaceManager(ctx)) return; + requireTreeOwner(ctx, treePath); +} + +async function grantSet( + params: GrantSetParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGrantAuthority(ctx, params.principalId, params.treePath); + await guardCore(() => + ctx.core.grantTreeAccess( + ctx.space.id, + params.principalId, + params.treePath, + params.access, + ), + ); + return { granted: true }; +} + +async function grantRemove( + params: GrantRemoveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGrantAuthority(ctx, params.principalId, params.treePath); + const removed = await guardCore(() => + ctx.core.removeTreeAccessGrant( + ctx.space.id, + params.principalId, + params.treePath, + ), + ); + return { removed }; +} + +async function grantList( + params: GrantListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Authorized when listing your OWN agent's grants, or a subtree you own, or + // (broadly) as a space manager. + const ownAgent = + params.principalId !== undefined && + params.principalId !== null && + (await callerOwnsAgent(ctx, params.principalId)); + const pathOwner = + params.treePath !== undefined && + params.treePath !== null && + ownsTreePath(ctx, params.treePath); + if (!ownAgent && !pathOwner) { + requireSpaceManager(ctx); + } + const grants = await ctx.core.listTreeAccessGrants( + ctx.space.id, + params.principalId ?? undefined, + params.treePath ?? undefined, + ); + return { grants: grants.map(toTreeGrantResponse) }; +} + +export const grantMethods = buildRegistry() + .register("grant.set", grantSetParams, grantSet) + .register("grant.remove", grantRemoveParams, grantRemove) + .register("grant.list", grantListParams, grantList) + .build(); diff --git a/packages/server/rpc/memory/group.ts b/packages/server/rpc/memory/group.ts new file mode 100644 index 0000000..486241f --- /dev/null +++ b/packages/server/rpc/memory/group.ts @@ -0,0 +1,182 @@ +/** + * Group handlers (group.*). Groups are space-scoped principals that bundle + * members for tree-access grants; group membership confers space access. + */ +import type { + GroupAddMemberParams, + GroupAddMemberResult, + GroupCreateParams, + GroupCreateResult, + GroupDeleteParams, + GroupDeleteResult, + GroupListForMemberParams, + GroupListForMemberResult, + GroupListMembersParams, + GroupListMembersResult, + GroupListParams, + GroupListResult, + GroupRemoveMemberParams, + GroupRemoveMemberResult, + GroupRenameParams, + GroupRenameResult, +} from "@memory.build/protocol/space"; +import { + groupAddMemberParams, + groupCreateParams, + groupDeleteParams, + groupListForMemberParams, + groupListMembersParams, + groupListParams, + groupRemoveMemberParams, + groupRenameParams, +} from "@memory.build/protocol/space"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + guardCore, + requireGroupAdmin, + requireSpaceAdmin, + toGroupMemberResponse, + toGroupMembershipResponse, + toGroupResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +/** Guard that the group exists in this space. */ +async function assertGroupInSpace( + ctx: SpaceRpcContext, + groupId: string, +): Promise { + const groups = await ctx.core.listSpaceGroups(ctx.space.id); + if (!groups.some((g) => g.id === groupId)) { + throw new AppError( + "NOT_FOUND", + `Group not found in this space: ${groupId}`, + ); + } +} + +async function groupCreate( + params: GroupCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const id = await guardCore(() => + ctx.core.createGroup(ctx.space.id, params.name), + ); + return { id }; +} + +async function groupList( + _params: GroupListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const groups = await ctx.core.listSpaceGroups(ctx.space.id); + return { groups: groups.map(toGroupResponse) }; +} + +async function groupRename( + params: GroupRenameParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + await assertGroupInSpace(ctx, params.id); + const renamed = await guardCore(() => + ctx.core.renamePrincipal(params.id, params.name), + ); + return { renamed }; +} + +async function groupDelete( + params: GroupDeleteParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + await assertGroupInSpace(ctx, params.id); + const deleted = await guardCore(() => ctx.core.deletePrincipal(params.id)); + return { deleted }; +} + +async function groupAddMember( + params: GroupAddMemberParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGroupAdmin(ctx, params.groupId); + await assertGroupInSpace(ctx, params.groupId); + await guardCore(() => + ctx.core.addGroupMember( + ctx.space.id, + params.groupId, + params.memberId, + params.admin ?? false, + ), + ); + return { added: true }; +} + +async function groupRemoveMember( + params: GroupRemoveMemberParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGroupAdmin(ctx, params.groupId); + await assertGroupInSpace(ctx, params.groupId); + const removed = await guardCore(() => + ctx.core.removeGroupMember(ctx.space.id, params.groupId, params.memberId), + ); + return { removed }; +} + +async function groupListMembers( + params: GroupListMembersParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGroupAdmin(ctx, params.groupId); + await assertGroupInSpace(ctx, params.groupId); + const members = await ctx.core.listGroupMembers(ctx.space.id, params.groupId); + return { members: members.map(toGroupMemberResponse) }; +} + +async function groupListForMember( + params: GroupListForMemberParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Anyone may see their OWN group memberships; seeing another principal's + // requires space-admin authority. + if (params.memberId !== ctx.principalId) { + requireSpaceAdmin(ctx); + } + const groups = await ctx.core.listGroupsForMember( + ctx.space.id, + params.memberId, + ); + return { groups: groups.map(toGroupMembershipResponse) }; +} + +export const groupMethods = buildRegistry() + .register("group.create", groupCreateParams, groupCreate) + .register("group.list", groupListParams, groupList) + .register("group.rename", groupRenameParams, groupRename) + .register("group.delete", groupDeleteParams, groupDelete) + .register("group.addMember", groupAddMemberParams, groupAddMember) + .register("group.removeMember", groupRemoveMemberParams, groupRemoveMember) + .register("group.listMembers", groupListMembersParams, groupListMembers) + .register("group.listForMember", groupListForMemberParams, groupListForMember) + .build(); diff --git a/packages/server/rpc/memory/index.ts b/packages/server/rpc/memory/index.ts index 593721f..181b8e9 100644 --- a/packages/server/rpc/memory/index.ts +++ b/packages/server/rpc/memory/index.ts @@ -1,13 +1,31 @@ /** * Memory RPC method registry — served at `/api/v1/memory/rpc`. * - * The new-model replacement for the engine RPC: memory data-plane methods - * (spaceStore) and, in 4C-2, space management methods (coreStore). Memory.* - * methods are wired here; management methods are added in Phase 4C-2. + * The new-model replacement for the engine RPC, combining the memory data-plane + * methods (spaceStore) with the space management methods (coreStore): membership, + * agents, groups, tree-access grants, and agent api keys. */ -export { memoryMethods } from "./memory"; +import type { MethodRegistry } from "../types"; +import { apiKeyMethods } from "./api-key"; +import { grantMethods } from "./grant"; +import { groupMethods } from "./group"; +import { memberMethods } from "./member"; +import { memoryDataMethods } from "./memory"; + export { assertSpaceRpcContext, isSpaceRpcContext, type SpaceRpcContext, } from "./types"; + +/** + * The full memory-endpoint registry: data-plane + space management methods. + * (Agent lifecycle lives on the user endpoint — see rpc/user.) + */ +export const memoryMethods: MethodRegistry = new Map([ + ...memoryDataMethods, + ...memberMethods, + ...groupMethods, + ...grantMethods, + ...apiKeyMethods, +]); diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts new file mode 100644 index 0000000..c3799b1 --- /dev/null +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -0,0 +1,493 @@ +// Integration test for the space management handlers (4C-2b): member / agent / +// group / grant / apiKey, driven through the merged memory registry against a +// provisioned space. The provisioned owner has owner@root, satisfying the +// management authorization gate. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/memory/management.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import { core as engineCore, space as engineSpace } from "@memory.build/engine"; +import type { TreeAccess } from "@memory.build/engine/core"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import { provisionUser } from "../../provision"; +import type { HandlerContext } from "../types"; +import { memoryMethods } from "./index"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +let ownerTreeAccess: TreeAccess; +let space: { id: string; slug: string }; +let ownerId: string; +let ownerEmail: string; + +function call( + method: string, + params: unknown, + as: { principalId?: string; treeAccess?: TreeAccess; admin?: boolean } = {}, +): Promise { + const registered = memoryMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/memory/rpc"), + store: engineSpace.spaceStore(sql, `me_${space.slug}`), + core: engineCore.coreStore(sql, coreSchema), + space, + principalId: as.principalId ?? ownerId, + apiKeyId: null, + treeAccess: as.treeAccess ?? ownerTreeAccess, + // the provisioned owner is also a space admin; non-owner callers default false + admin: as.admin ?? as.principalId === undefined, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +/** Create a standalone global user (no auth), returning its id. */ +async function makeUser(): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await engineCore + .coreStore(sql, coreSchema) + .createUser(id, `u_${rand(8)}@example.com`); + return id; +} + +/** + * Create a global agent owned by `owner` (the user-endpoint operation), returning + * its id. Not yet a member of any space — member.add brings it in. + */ +function makeAgent(owner: string): Promise { + return engineCore + .coreStore(sql, coreSchema) + .createAgent(owner, `agent_${rand(6)}`); +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand(8)}`; + coreSchema = `core_test_${rand(8)}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + ownerEmail = `owner_${crypto.randomUUID().slice(0, 8)}@example.com`; + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: ownerEmail, + name: "Owner", + provider: "github", + accountId: crypto.randomUUID(), + }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + space = { id: r.spaceId, slug: r.spaceSlug }; + ownerId = r.userId; + ownerTreeAccess = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(r.userId, r.spaceId); +}); + +test("member: list / resolveByEmail / add / remove", async () => { + const listed = await call<{ members: { id: string; admin: boolean }[] }>( + "member.list", + {}, + ); + expect(listed.members.some((m) => m.id === ownerId && m.admin)).toBe(true); + + const resolved = await call<{ principal: { id: string } | null }>( + "member.resolveByEmail", + { email: ownerEmail }, + ); + expect(resolved.principal?.id).toBe(ownerId); + + const other = await makeUser(); + expect( + (await call<{ added: boolean }>("member.add", { principalId: other })) + .added, + ).toBe(true); + expect( + (await call<{ members: { id: string }[] }>("member.list", {})).members.some( + (m) => m.id === other, + ), + ).toBe(true); + expect( + (await call<{ removed: boolean }>("member.remove", { principalId: other })) + .removed, + ).toBe(true); +}); + +test("group: create / list / members / rename / delete", async () => { + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "eng", + }); + expect( + (await call<{ groups: { id: string }[] }>("group.list", {})).groups.some( + (g) => g.id === groupId, + ), + ).toBe(true); + + await call("group.addMember", { groupId, memberId: ownerId, admin: true }); + const members = await call<{ + members: { memberId: string; admin: boolean }[]; + }>("group.listMembers", { groupId }); + expect(members.members[0]?.memberId).toBe(ownerId); + expect(members.members[0]?.admin).toBe(true); + + const forMember = await call<{ groups: { groupId: string }[] }>( + "group.listForMember", + { memberId: ownerId }, + ); + expect(forMember.groups.some((g) => g.groupId === groupId)).toBe(true); + + expect( + ( + await call<{ renamed: boolean }>("group.rename", { + id: groupId, + name: "platform", + }) + ).renamed, + ).toBe(true); + expect( + ( + await call<{ removed: boolean }>("group.removeMember", { + groupId, + memberId: ownerId, + }) + ).removed, + ).toBe(true); + expect( + (await call<{ deleted: boolean }>("group.delete", { id: groupId })).deleted, + ).toBe(true); +}); + +test("grant: set / list / remove", async () => { + const other = await makeUser(); + await call("grant.set", { principalId: other, treePath: "docs", access: 1 }); + const grants = await call<{ + grants: { principalId: string; treePath: string; access: number }[]; + }>("grant.list", { principalId: other }); + expect(grants.grants).toHaveLength(1); + expect(grants.grants[0]?.treePath).toBe("docs"); + expect(grants.grants[0]?.access).toBe(1); + + expect( + ( + await call<{ removed: boolean }>("grant.remove", { + principalId: other, + treePath: "docs", + }) + ).removed, + ).toBe(true); +}); + +test("apiKey: create (agent-only) / list / get / delete", async () => { + // agent lifecycle is the user endpoint's job; here the owner brings an agent + // into the space, then mints its (space-bound) key. + const agentId = await makeAgent(ownerId); + await call("member.add", { principalId: agentId }); + const created = await call<{ id: string; key: string }>("apiKey.create", { + agentId, + name: "ci", + }); + expect(created.key.startsWith(`me.${space.slug}.`)).toBe(true); + + const list = await call<{ apiKeys: { id: string }[] }>("apiKey.list", { + memberId: agentId, + }); + expect(list.apiKeys.map((k) => k.id)).toContain(created.id); + + const got = await call<{ apiKey: { id: string } | null }>("apiKey.get", { + id: created.id, + }); + expect(got.apiKey?.id).toBe(created.id); + + expect( + (await call<{ deleted: boolean }>("apiKey.delete", { id: created.id })) + .deleted, + ).toBe(true); + expect( + (await call<{ apiKey: unknown }>("apiKey.get", { id: created.id })).apiKey, + ).toBeNull(); +}); + +test("apiKey.create rejects a non-agent member", async () => { + // ownerId is a user, not an agent → NOT_FOUND in this space's agents + await expectAppError( + call("apiKey.create", { agentId: ownerId, name: "nope" }), + "NOT_FOUND", + ); +}); + +test("roster/group management requires admin or owner", async () => { + // a plain member: write access on a subtree, not an admin, not a root owner + const member = await makeUser(); + const as = { + principalId: member, + treeAccess: [{ tree_path: "sub", access: 2 }] as TreeAccess, + admin: false, + }; + await expectAppError(call("member.list", {}, as), "FORBIDDEN"); + await expectAppError(call("group.create", { name: "x" }, as), "FORBIDDEN"); +}); + +test("a space admin (without owner@root) has management authority", async () => { + // an admin member with only read access, no owner grant anywhere + const adminMember = await makeUser(); + await call("member.add", { principalId: adminMember, admin: true }); + await call("grant.set", { + principalId: adminMember, + treePath: "x", + access: 1, + }); + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(adminMember, space.id); + const as = { principalId: adminMember, treeAccess: ta, admin: true }; + + // can manage the roster and grant anywhere despite holding no owner grant + expect( + (await call<{ members: unknown[] }>("member.list", {}, as)).members.length, + ).toBeGreaterThan(0); + const stranger = await makeUser(); + expect( + ( + await call<{ granted: boolean }>( + "grant.set", + { principalId: stranger, treePath: "anywhere", access: 2 }, + as, + ) + ).granted, + ).toBe(true); +}); + +test("group.listForMember: own memberships are self-service, others need admin", async () => { + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "squad", + }); + const member = await makeUser(); + await call("group.addMember", { groupId, memberId: member }); + const as = { + principalId: member, + treeAccess: [] as TreeAccess, + admin: false, + }; + + // the member can see their own memberships + const mine = await call<{ groups: { groupId: string }[] }>( + "group.listForMember", + { memberId: member }, + as, + ); + expect(mine.groups.some((g) => g.groupId === groupId)).toBe(true); + + // but not someone else's + await expectAppError( + call("group.listForMember", { memberId: ownerId }, as), + "FORBIDDEN", + ); +}); + +test("group member management allows a group admin (not a space admin)", async () => { + // owner creates a group and makes `lead` an admin of it + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "team", + }); + const lead = await makeUser(); + // lead is only a group admin (not added to principal_space) — group + // membership is transitive, so this is enough authority over the group + await call("group.addMember", { groupId, memberId: lead, admin: true }); + const as = { principalId: lead, treeAccess: [] as TreeAccess, admin: false }; + + // lead can manage THIS group's membership + const other = await makeUser(); + expect( + ( + await call<{ added: boolean }>( + "group.addMember", + { groupId, memberId: other }, + as, + ) + ).added, + ).toBe(true); + const members = await call<{ members: { memberId: string }[] }>( + "group.listMembers", + { groupId }, + as, + ); + expect(members.members.some((m) => m.memberId === other)).toBe(true); + expect( + ( + await call<{ removed: boolean }>( + "group.removeMember", + { groupId, memberId: other }, + as, + ) + ).removed, + ).toBe(true); + + // but structural ops (create) still require a space admin + await expectAppError(call("group.create", { name: "nope" }, as), "FORBIDDEN"); + + // and a non-admin, non-member can't manage the group + const stranger = await makeUser(); + await expectAppError( + call( + "group.listMembers", + { groupId }, + { principalId: stranger, treeAccess: [] as TreeAccess, admin: false }, + ), + "FORBIDDEN", + ); +}); + +test("group management requires admin — owner@root is not enough", async () => { + // a member who owns the whole data tree (owner@root) but is NOT a space admin + const member = await makeUser(); + await call("member.add", { principalId: member }); + await call("grant.set", { principalId: member, treePath: "", access: 3 }); + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(member, space.id); + const as = { principalId: member, treeAccess: ta, admin: false }; + + // owner@root can manage the roster and grant access (it's their data) + expect( + (await call<{ members: unknown[] }>("member.list", {}, as)).members.length, + ).toBeGreaterThan(0); + // but groups are structural — admin only + await expectAppError(call("group.create", { name: "g" }, as), "FORBIDDEN"); +}); + +test("grant authority is path-scoped: a subtree owner delegates within it", async () => { + // a member who owns "proj" (not the root) can manage access under proj only + const member = await makeUser(); + await call("member.add", { principalId: member }); + await call("grant.set", { principalId: member, treePath: "proj", access: 3 }); + const memberTA = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(member, space.id); + const as = { principalId: member, treeAccess: memberTA }; + + const stranger = await makeUser(); + // within the owned subtree → allowed + expect( + ( + await call<{ granted: boolean }>( + "grant.set", + { principalId: stranger, treePath: "proj.sub", access: 1 }, + as, + ) + ).granted, + ).toBe(true); + // outside it → FORBIDDEN + await expectAppError( + call( + "grant.set", + { principalId: stranger, treePath: "other", access: 1 }, + as, + ), + "FORBIDDEN", + ); + + // can list grants under the owned subtree, but not the whole space + const underProj = await call<{ + grants: { treePath: string }[]; + }>("grant.list", { treePath: "proj" }, as); + expect(underProj.grants.some((g) => g.treePath === "proj.sub")).toBe(true); + await expectAppError(call("grant.list", {}, as), "FORBIDDEN"); +}); + +test("self-service: a non-owner member brings their own agent into the space", async () => { + // owner onboards a second user with write (not owner) on a subtree + const member = await makeUser(); + await call("member.add", { principalId: member }); + await call("grant.set", { principalId: member, treePath: "proj", access: 2 }); + const memberTA = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(member, space.id); + const as = { principalId: member, treeAccess: memberTA }; + + // the member created their agent on the user endpoint (simulated via core); + // they bring it into the space (self-service member.add) without owner rights + const agentId = await makeAgent(member); + expect( + (await call<{ added: boolean }>("member.add", { principalId: agentId }, as)) + .added, + ).toBe(true); + + // and mint its key + self-grant it (capped by their own access) + const key = await call<{ key: string }>( + "apiKey.create", + { agentId, name: "k" }, + as, + ); + expect(key.key.startsWith(`me.${space.slug}.`)).toBe(true); + expect( + ( + await call<{ granted: boolean }>( + "grant.set", + { principalId: agentId, treePath: "proj", access: 2 }, + as, + ) + ).granted, + ).toBe(true); + + // but the member cannot manage the roster, add a stranger, or grant to others + await expectAppError(call("member.list", {}, as), "FORBIDDEN"); + const stranger = await makeUser(); + await expectAppError( + call("member.add", { principalId: stranger }, as), + "FORBIDDEN", + ); + await expectAppError( + call( + "grant.set", + { principalId: stranger, treePath: "proj", access: 1 }, + as, + ), + "FORBIDDEN", + ); +}); diff --git a/packages/server/rpc/memory/member.ts b/packages/server/rpc/memory/member.ts new file mode 100644 index 0000000..76974ea --- /dev/null +++ b/packages/server/rpc/memory/member.ts @@ -0,0 +1,103 @@ +/** + * Space membership handlers (member.*) — the space roster (principal_space). + */ +import type { + MemberAddParams, + MemberAddResult, + MemberListParams, + MemberListResult, + MemberRemoveParams, + MemberRemoveResult, + MemberResolveByEmailParams, + MemberResolveByEmailResult, +} from "@memory.build/protocol/space"; +import { + memberAddParams, + memberListParams, + memberRemoveParams, + memberResolveByEmailParams, +} from "@memory.build/protocol/space"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + callerOwnsAgentGlobal, + guardCore, + requireSpaceManager, + toPrincipalResponse, + toSpaceMemberResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +async function memberList( + params: MemberListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceManager(ctx); + const members = await ctx.core.listSpaceMembers( + ctx.space.id, + params.kind ?? undefined, + ); + return { members: members.map(toSpaceMemberResponse) }; +} + +async function memberAdd( + params: MemberAddParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Bringing your OWN agent into a space is self-service (it stays capped by + // your access); adding anyone else requires space-owner authority. A member + // can't grant themselves admin on their own agent membership. + const ownAgent = + params.admin !== true && + (await callerOwnsAgentGlobal(ctx, params.principalId)); + if (!ownAgent) { + requireSpaceManager(ctx); + } + await guardCore(() => + ctx.core.addPrincipalToSpace( + ctx.space.id, + params.principalId, + params.admin ?? false, + ), + ); + return { added: true }; +} + +async function memberRemove( + params: MemberRemoveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceManager(ctx); + const removed = await guardCore(() => + ctx.core.removePrincipalFromSpace(ctx.space.id, params.principalId), + ); + return { removed }; +} + +async function memberResolveByEmail( + params: MemberResolveByEmailParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceManager(ctx); + const principal = await ctx.core.getUserByName(params.email); + return { principal: principal ? toPrincipalResponse(principal) : null }; +} + +export const memberMethods = buildRegistry() + .register("member.list", memberListParams, memberList) + .register("member.add", memberAddParams, memberAdd) + .register("member.remove", memberRemoveParams, memberRemove) + .register( + "member.resolveByEmail", + memberResolveByEmailParams, + memberResolveByEmail, + ) + .build(); diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index dd69634..07f5d4d 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -20,7 +20,7 @@ import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; import postgres, { type Sql } from "postgres"; import { provisionUser } from "../../provision"; import type { HandlerContext } from "../types"; -import { memoryMethods } from "./memory"; +import { memoryDataMethods } from "./memory"; const URL = process.env.TEST_DATABASE_URL ?? @@ -50,7 +50,7 @@ function call( params: unknown, ta: TreeAccess = treeAccess, ): Promise { - const registered = memoryMethods.get(method); + const registered = memoryDataMethods.get(method); if (!registered) throw new Error(`no handler for ${method}`); const context = { request: new Request("http://localhost/api/v1/memory/rpc"), diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index fa1b2f4..001db26 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -455,7 +455,7 @@ async function memoryCountTree( // Registry // ============================================================================= -export const memoryMethods = buildRegistry() +export const memoryDataMethods = buildRegistry() .register("memory.create", memoryCreateParams, memoryCreate) .register("memory.batchCreate", memoryBatchCreateParams, memoryBatchCreate) .register("memory.get", memoryGetParams, memoryGet) diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts new file mode 100644 index 0000000..ca17e09 --- /dev/null +++ b/packages/server/rpc/memory/support.ts @@ -0,0 +1,259 @@ +/** + * Shared helpers for the space management handlers (member/agent/group/grant/ + * apiKey): the owner authorization gate, core SQL error mapping, and response + * serializers. + */ + +import type { + ApiKeyInfo, + Group, + GroupMember, + GroupMembership, + Principal, + SpaceMember, + TreeGrant, +} from "@memory.build/engine/core"; +import { ACCESS, ROOT_PATH } from "@memory.build/engine/core"; +import type { + ApiKeyInfoResponse, + GroupMemberResponse, + GroupMembershipResponse, + GroupResponse, + PrincipalResponse, + SpaceMemberResponse, + TreeGrantResponse, +} from "@memory.build/protocol/space"; +import { guardCore } from "../core-error"; +import { AppError } from "../errors"; +import type { SpaceRpcContext } from "./types"; + +export { guardCore }; + +/** Owner-level grant (3) at the space root — owns the whole space. */ +export function isSpaceOwner(context: SpaceRpcContext): boolean { + return context.treeAccess.some( + (g) => g.tree_path === ROOT_PATH && g.access >= ACCESS.owner, + ); +} + +/** + * Structural authority over the space (principal_space.admin). Required for + * managing groups — a structural construct of the space, distinct from data + * ownership: owning the data tree (owner@root) is NOT sufficient. + */ +export function requireSpaceAdmin(context: SpaceRpcContext): void { + if (!context.admin) { + throw new AppError("FORBIDDEN", "This action requires being a space admin"); + } +} + +/** + * Authority to manage a group's membership: a space admin, or an admin of the + * group itself (group_member.admin). Used by group.addMember / removeMember / + * listMembers. (Creating/renaming/deleting groups stays space-admin only.) + */ +export async function requireGroupAdmin( + context: SpaceRpcContext, + groupId: string, +): Promise { + if (context.admin) return; + const groupAdmin = await context.core.isGroupAdmin( + context.principalId, + groupId, + context.space.id, + ); + if (!groupAdmin) { + throw new AppError( + "FORBIDDEN", + "Managing group members requires being a space admin or an admin of the group", + ); + } +} + +/** + * Space-management authority: a space admin (principal_space.admin) or the + * space owner (owner@root). Gates roster management and broad grant listing — + * controlling access to data the owner owns. (Per-subtree grant delegation is + * handled by requireTreeOwner; group structure requires requireSpaceAdmin.) + */ +export function isSpaceManager(context: SpaceRpcContext): boolean { + return context.admin || isSpaceOwner(context); +} + +export function requireSpaceManager(context: SpaceRpcContext): void { + if (!isSpaceManager(context)) { + throw new AppError( + "FORBIDDEN", + "Space management requires being a space admin or owner", + ); + } +} + +/** True if `ancestor` is an ancestor-or-self of `path` (ltree `@>`). */ +function isAncestorOrSelf(ancestor: string, path: string): boolean { + return ( + ancestor === ROOT_PATH || + path === ancestor || + path.startsWith(`${ancestor}.`) + ); +} + +/** + * Owner authority at a specific tree path: the caller holds an owner grant (3) + * at the path or any ancestor of it. This is how grants are delegated — owning + * a subtree lets you manage access within it. Owner@root is the case that + * covers the whole space. + */ +export function ownsTreePath( + context: SpaceRpcContext, + treePath: string, +): boolean { + return context.treeAccess.some( + (g) => g.access >= ACCESS.owner && isAncestorOrSelf(g.tree_path, treePath), + ); +} + +export function requireTreeOwner( + context: SpaceRpcContext, + treePath: string, +): void { + if (!ownsTreePath(context, treePath)) { + throw new AppError( + "FORBIDDEN", + `Granting access at "${treePath}" requires owner access on that path`, + ); + } +} + +/** + * True if `principalId` is an agent in this space owned by the caller. Agents + * are user-owned and capped by their owner's access (agent_tree_access), so a + * member managing their own agents (create/keys/self-grants) is self-service + * and safe — it can't escalate beyond the caller's own access. + */ +export async function callerOwnsAgent( + context: SpaceRpcContext, + principalId: string, +): Promise { + const agents = await context.core.listSpaceMembers(context.space.id, "a"); + const agent = agents.find((a) => a.id === principalId); + return agent !== undefined && agent.ownerId === context.principalId; +} + +/** + * Assert the caller owns `agentId` (an agent in this space). NOT_FOUND if the + * agent isn't a member of this space; FORBIDDEN if it's owned by someone else. + */ +export async function requireOwnedAgent( + context: SpaceRpcContext, + agentId: string, +): Promise { + const agents = await context.core.listSpaceMembers(context.space.id, "a"); + const agent = agents.find((a) => a.id === agentId); + if (!agent) { + throw new AppError( + "NOT_FOUND", + `Agent not found in this space: ${agentId}`, + ); + } + if (agent.ownerId !== context.principalId) { + throw new AppError("FORBIDDEN", "Not the owner of this agent"); + } +} + +/** + * True if `principalId` is an agent owned by the caller, checked globally (not + * scoped to the current space). Used by member.add so a member can bring their + * OWN agent into a space before it is a member there. + */ +export async function callerOwnsAgentGlobal( + context: SpaceRpcContext, + principalId: string, +): Promise { + const principal = await context.core.getPrincipal(principalId); + return ( + principal !== null && + principal.kind === "a" && + principal.ownerId === context.principalId + ); +} + +// ============================================================================= +// Serializers (Date → ISO) +// ============================================================================= + +export function toSpaceMemberResponse(m: SpaceMember): SpaceMemberResponse { + return { + id: m.id, + kind: m.kind, + name: m.name, + ownerId: m.ownerId, + direct: m.direct, + admin: m.admin, + createdAt: m.createdAt.toISOString(), + updatedAt: m.updatedAt?.toISOString() ?? null, + }; +} + +export function toPrincipalResponse(p: Principal): PrincipalResponse { + return { + id: p.id, + kind: p.kind, + name: p.name, + ownerId: p.ownerId, + spaceId: p.spaceId, + createdAt: p.createdAt.toISOString(), + updatedAt: p.updatedAt?.toISOString() ?? null, + }; +} + +export function toGroupResponse(g: Group): GroupResponse { + return { + id: g.id, + name: g.name, + createdAt: g.createdAt.toISOString(), + updatedAt: g.updatedAt?.toISOString() ?? null, + }; +} + +export function toGroupMemberResponse(m: GroupMember): GroupMemberResponse { + return { + memberId: m.memberId, + kind: m.kind, + name: m.name, + admin: m.admin, + createdAt: m.createdAt.toISOString(), + }; +} + +export function toGroupMembershipResponse( + m: GroupMembership, +): GroupMembershipResponse { + return { + groupId: m.groupId, + name: m.name, + admin: m.admin, + createdAt: m.createdAt.toISOString(), + }; +} + +export function toTreeGrantResponse(g: TreeGrant): TreeGrantResponse { + return { + principalId: g.principalId, + treePath: g.treePath, + access: g.access, + createdAt: g.createdAt.toISOString(), + updatedAt: g.updatedAt?.toISOString() ?? null, + }; +} + +export function toApiKeyInfoResponse(k: ApiKeyInfo): ApiKeyInfoResponse { + return { + id: k.id, + memberId: k.memberId, + lookupId: k.lookupId, + name: k.name, + createdAt: k.createdAt.toISOString(), + expiresAt: k.expiresAt?.toISOString() ?? null, + }; +} diff --git a/packages/server/rpc/memory/types.ts b/packages/server/rpc/memory/types.ts index da95982..b6c3420 100644 --- a/packages/server/rpc/memory/types.ts +++ b/packages/server/rpc/memory/types.ts @@ -23,6 +23,8 @@ export interface SpaceRpcContext extends HandlerContext { apiKeyId: string | null; /** The principal's effective grants in this space — the access gate. */ treeAccess: TreeAccess; + /** Whether the principal is a space admin (principal_space.admin). */ + admin: boolean; /** Embedding config for semantic search (optional). */ embeddingConfig?: EmbeddingConfig; } diff --git a/packages/server/rpc/user/agent.integration.test.ts b/packages/server/rpc/user/agent.integration.test.ts new file mode 100644 index 0000000..864b6db --- /dev/null +++ b/packages/server/rpc/user/agent.integration.test.ts @@ -0,0 +1,127 @@ +// Integration test for the user RPC agent handlers (agent.* lifecycle). +// User-scoped (no space): a user manages their own global service accounts. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/user/agent.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { migrateCore } from "@memory.build/database"; +import { coreStore } from "@memory.build/engine/core"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import type { HandlerContext } from "../types"; +import { userMethods } from "./index"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let coreSchema: string; +let userId: string; + +function call( + method: string, + params: unknown, + asUser: string = userId, +): Promise { + const registered = userMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/user/rpc"), + core: coreStore(sql, coreSchema), + userId: asUser, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +async function makeUser(): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await coreStore(sql, coreSchema).createUser(id, `u_${rand(8)}@example.com`); + return id; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + userId = await makeUser(); +}); + +test("create / list / rename / delete the caller's agents", async () => { + const { id } = await call<{ id: string }>("agent.create", { name: "bot" }); + + let agents = await call<{ agents: { id: string; name: string }[] }>( + "agent.list", + {}, + ); + expect(agents.agents).toHaveLength(1); + expect(agents.agents[0]?.id).toBe(id); + expect(agents.agents[0]?.name).toBe("bot"); + + expect( + (await call<{ renamed: boolean }>("agent.rename", { id, name: "bot2" })) + .renamed, + ).toBe(true); + agents = await call("agent.list", {}); + expect(agents.agents[0]?.name).toBe("bot2"); + + expect( + (await call<{ deleted: boolean }>("agent.delete", { id })).deleted, + ).toBe(true); + expect( + (await call<{ agents: unknown[] }>("agent.list", {})).agents, + ).toHaveLength(0); +}); + +test("agent.list is scoped to the caller", async () => { + await call("agent.create", { name: "mine" }); + const other = await makeUser(); + const otherList = await call<{ agents: unknown[] }>("agent.list", {}, other); + expect(otherList.agents).toHaveLength(0); +}); + +test("cannot rename/delete another user's agent", async () => { + const { id } = await call<{ id: string }>("agent.create", { name: "mine" }); + const intruder = await makeUser(); + await expectAppError( + call("agent.rename", { id, name: "hijacked" }, intruder), + "FORBIDDEN", + ); + await expectAppError(call("agent.delete", { id }, intruder), "FORBIDDEN"); +}); + +test("rename/delete of a non-existent agent → NOT_FOUND", async () => { + const [row] = await sql`select uuidv7() as id`; + const ghost = row?.id as string; + await expectAppError( + call("agent.rename", { id: ghost, name: "x" }), + "NOT_FOUND", + ); +}); diff --git a/packages/server/rpc/user/agent.ts b/packages/server/rpc/user/agent.ts new file mode 100644 index 0000000..611e9e0 --- /dev/null +++ b/packages/server/rpc/user/agent.ts @@ -0,0 +1,107 @@ +/** + * Agent handlers (agent.*) for the user RPC. + * + * Agents are a user's global service accounts. The lifecycle here is purely + * user-scoped: create / list / rename / delete the caller's own agents. + * Bringing an agent into a space (member.add) and minting its space-bound api + * key (apiKey.create) are space-endpoint operations. + */ +import type { Principal } from "@memory.build/engine/core"; +import type { + AgentCreateParams, + AgentCreateResult, + AgentDeleteParams, + AgentDeleteResult, + AgentListParams, + AgentListResult, + AgentRenameParams, + AgentRenameResult, + AgentResponse, +} from "@memory.build/protocol/user"; +import { + agentCreateParams, + agentDeleteParams, + agentListParams, + agentRenameParams, +} from "@memory.build/protocol/user"; +import { guardCore } from "../core-error"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +function toAgentResponse(p: Principal): AgentResponse { + return { + id: p.id, + name: p.name, + createdAt: p.createdAt.toISOString(), + updatedAt: p.updatedAt?.toISOString() ?? null, + }; +} + +/** Assert the caller owns this agent (globally). */ +async function requireOwnAgent( + ctx: UserRpcContext, + agentId: string, +): Promise { + const principal = await ctx.core.getPrincipal(agentId); + if (!principal || principal.kind !== "a") { + throw new AppError("NOT_FOUND", `Agent not found: ${agentId}`); + } + if (principal.ownerId !== ctx.userId) { + throw new AppError("FORBIDDEN", "Not the owner of this agent"); + } +} + +async function agentCreate( + params: AgentCreateParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const id = await guardCore(() => + ctx.core.createAgent(ctx.userId, params.name), + ); + return { id }; +} + +async function agentList( + _params: AgentListParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const agents = await ctx.core.listAgents(ctx.userId); + return { agents: agents.map(toAgentResponse) }; +} + +async function agentRename( + params: AgentRenameParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireOwnAgent(ctx, params.id); + const renamed = await guardCore(() => + ctx.core.renamePrincipal(params.id, params.name), + ); + return { renamed }; +} + +async function agentDelete( + params: AgentDeleteParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireOwnAgent(ctx, params.id); + const deleted = await guardCore(() => ctx.core.deletePrincipal(params.id)); + return { deleted }; +} + +export const agentMethods = buildRegistry() + .register("agent.create", agentCreateParams, agentCreate) + .register("agent.list", agentListParams, agentList) + .register("agent.rename", agentRenameParams, agentRename) + .register("agent.delete", agentDeleteParams, agentDelete) + .build(); diff --git a/packages/server/rpc/user/index.ts b/packages/server/rpc/user/index.ts new file mode 100644 index 0000000..7783693 --- /dev/null +++ b/packages/server/rpc/user/index.ts @@ -0,0 +1,13 @@ +/** + * User RPC method registry — served at `/api/v1/user/rpc` (session-only, + * user-scoped). Currently the lifecycle of a user's agents. + */ +import { agentMethods } from "./agent"; + +export { + assertUserRpcContext, + isUserRpcContext, + type UserRpcContext, +} from "./types"; + +export const userMethods = agentMethods; diff --git a/packages/server/rpc/user/types.ts b/packages/server/rpc/user/types.ts new file mode 100644 index 0000000..6268fc7 --- /dev/null +++ b/packages/server/rpc/user/types.ts @@ -0,0 +1,31 @@ +/** + * User RPC context — populated by authenticateUser. User-scoped (no space): + * the calling user manages their own global service accounts (agents). + */ +import type { CoreStore } from "@memory.build/engine/core"; +import type { HandlerContext } from "../types"; + +export interface UserRpcContext extends HandlerContext { + /** Core control-plane store. */ + core: CoreStore; + /** The authenticated user id (== the core user-principal id). */ + userId: string; +} + +export function isUserRpcContext(ctx: HandlerContext): ctx is UserRpcContext { + return ( + "core" in ctx && + typeof ctx.core === "object" && + ctx.core !== null && + "userId" in ctx && + typeof ctx.userId === "string" + ); +} + +export function assertUserRpcContext( + ctx: HandlerContext, +): asserts ctx is UserRpcContext { + if (!isUserRpcContext(ctx)) { + throw new Error("User context not initialized (authentication required)"); + } +} From df0d40404a3dbb63069a3090128dc3ba17af30ad Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 3 Jun 2026 22:24:39 +0200 Subject: [PATCH 044/156] feat(worker): cut the embedding worker over to the space model (4D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repurpose the embedding worker to process per-space me_ schemas on the new-model postgres.js pool, replacing the legacy per-engine path. - driver: Bun.SQL → postgres.js throughout the worker; missing-schema detection duck-types SQLSTATE 3F000 off `.code`. - drop sharding (pgdog.shard) and `SET ROLE me_embed` — the new model is not sharded and the space functions are security-invoker, so the worker runs as the pool user; per-tx it only sets search_path + statement/lock/transaction timeouts. - write-back no longer references a max_attempts column (the space queue has none): transient failures just record last_error and leave outcome NULL; the claim function's sweep finalizes exhausted rows to 'failed'. - discovery: core.list_spaces() + coreStore.listSpaces(); the server builds a dedicated postgres.js worker pool and discovers spaces (slug → me_) instead of accountsDb.listActiveEngines(). - types: EngineTarget → SpaceTarget (no shard); local WorkerTimeouts replaces the engine/ops/_tx dependency; stats.enginesDropped → spacesDropped. - rewrite the worker integration test against a real space schema (migrateSpace) and the new claim-sweep failure semantics; update the unit test's mock error. Co-Authored-By: Claude Opus 4.8 (1M context) --- bun.lock | 3 +- .../core/migrate/idempotent/004_space.sql | 22 + packages/engine/core/db.ts | 7 + packages/server/index.ts | 43 +- packages/worker/index.ts | 12 +- packages/worker/package.json | 5 +- packages/worker/pool.ts | 12 +- packages/worker/process.integration.test.ts | 651 +++++------------- packages/worker/process.ts | 133 ++-- packages/worker/types.ts | 36 +- packages/worker/worker.test.ts | 31 +- packages/worker/worker.ts | 34 +- 12 files changed, 380 insertions(+), 609 deletions(-) diff --git a/bun.lock b/bun.lock index d0d3997..9b87aca 100644 --- a/bun.lock +++ b/bun.lock @@ -176,9 +176,10 @@ "name": "@memory.build/worker", "version": "0.2.5", "dependencies": { + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", - "@memory.build/engine": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", }, }, "scripts": { diff --git a/packages/database/core/migrate/idempotent/004_space.sql b/packages/database/core/migrate/idempotent/004_space.sql index 33548f9..cb31721 100644 --- a/packages/database/core/migrate/idempotent/004_space.sql +++ b/packages/database/core/migrate/idempotent/004_space.sql @@ -36,3 +36,25 @@ as $func$ $func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; + +------------------------------------------------------------------------------- +-- list_spaces +-- All spaces, newest first. Used by the embedding worker to discover the +-- me_ data schemas to process. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_spaces() +returns table +( id uuid +, slug text +, name text +, language text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select s.id, s.slug, s.name::text, s.language, s.created_at, s.updated_at + from {{schema}}.space s + order by s.created_at desc +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index 234885f..9c2df15 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -27,6 +27,8 @@ import type { export interface CoreStore { createSpace(slug: string, name: string, language?: string): Promise; getSpace(slug: string): Promise; + /** All spaces (e.g. for the embedding worker to discover me_ schemas). */ + listSpaces(): Promise; createUser(id: string, name: string): Promise; createAgent(ownerId: string, name: string, id?: string): Promise; @@ -201,6 +203,11 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return row ? mapSpace(row) : null; }, + async listSpaces() { + const rows = await sql`select * from ${sch}.list_spaces()`; + return rows.map(mapSpace); + }, + async createUser(id, name) { const [row] = await sql`select ${sch}.create_user(${id}, ${name}) as id`; if (!row) throw new Error("create_user returned no row"); diff --git a/packages/server/index.ts b/packages/server/index.ts index ba9ac08..b56c4ae 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -6,6 +6,7 @@ import { bootstrapSpaceDatabase, migrateAuth, migrateCore, + slugToSchema as spaceSlugToSchema, } from "@memory.build/database"; import type { EmbeddingConfig } from "@memory.build/embedding"; import { coreStore } from "@memory.build/engine/core"; @@ -16,10 +17,10 @@ import { import { bootstrap as bootstrapEngine } from "@memory.build/engine/migrate/bootstrap"; import { migrateAll as migrateEngines } from "@memory.build/engine/migrate/runner"; import { - DEFAULT_ENGINE_TIMEOUTS, - type EngineTimeouts, -} from "@memory.build/engine/ops/_tx"; -import { WorkerPool } from "@memory.build/worker"; + DEFAULT_WORKER_TIMEOUTS, + WorkerPool, + type WorkerTimeouts, +} from "@memory.build/worker"; import { configure, info, reportError, span } from "@pydantic/logfire-node"; import postgres from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; @@ -242,19 +243,19 @@ const workerEnginePoolConnectionTimeout = parseIntEnv( process.env.WORKER_ENGINE_POOL_CONNECTION_TIMEOUT || "", String(enginePoolConnectionTimeout), ); -const workerEngineTimeouts: EngineTimeouts = { +const workerTimeouts: WorkerTimeouts = { statementTimeout: process.env.WORKER_ENGINE_STATEMENT_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.statementTimeout, + DEFAULT_WORKER_TIMEOUTS.statementTimeout, lockTimeout: process.env.WORKER_ENGINE_LOCK_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.lockTimeout, + DEFAULT_WORKER_TIMEOUTS.lockTimeout, transactionTimeout: process.env.WORKER_ENGINE_TRANSACTION_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.transactionTimeout, + DEFAULT_WORKER_TIMEOUTS.transactionTimeout, idleInTransactionSessionTimeout: process.env.WORKER_ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.idleInTransactionSessionTimeout, + DEFAULT_WORKER_TIMEOUTS.idleInTransactionSessionTimeout, }; // ============================================================================= @@ -357,11 +358,14 @@ const engineSql = new Bun.SQL(engineDatabaseUrl, { connectionTimeout: enginePoolConnectionTimeout, }); -const workerEngineSql = new Bun.SQL(workerEngineDatabaseUrl, { +// Dedicated worker pool (postgres.js) on the new-model DB — the embedding +// worker processes the per-space me_ schemas that live there. +const workerDb = postgres(workerEngineDatabaseUrl, { max: workerEnginePoolMax, - idleTimeout: workerEnginePoolIdleReapSeconds, - maxLifetime: workerEnginePoolMaxLifetime, - connectionTimeout: workerEnginePoolConnectionTimeout, + idle_timeout: workerEnginePoolIdleReapSeconds, + max_lifetime: workerEnginePoolMaxLifetime, + connect_timeout: workerEnginePoolConnectionTimeout, + onnotice: () => {}, }); // New-model pool (postgres.js): the auth + core control plane and the per-space @@ -489,14 +493,11 @@ const router = createRouter(serverContext); // Embedding Worker Pool // ============================================================================= -const workerPool = new WorkerPool(workerEngineSql, { +const workerPool = new WorkerPool(workerDb, { embedding: embeddingConfig, discover: async () => { - const engines = await accountsDb.listActiveEngines(); - return engines.map((e) => ({ - schema: slugToSchema(e.slug), - shard: e.shardId, - })); + const spaces = await core.listSpaces(); + return spaces.map((s) => ({ schema: spaceSlugToSchema(s.slug) })); }, batchSize: parseIntEnv( "WORKER_BATCH_SIZE", @@ -519,7 +520,7 @@ const workerPool = new WorkerPool(workerEngineSql, { process.env.WORKER_REFRESH_INTERVAL_MS || "", "60000", ), - workerEngineTimeouts, + timeouts: workerTimeouts, }); await workerPool.start(workerCount); @@ -619,7 +620,7 @@ async function shutdown() { try { await accountsSql.close(); await engineSql.close(); - await workerEngineSql.close(); + await workerDb.end(); await db.end(); } catch (error) { reportError("Error closing database connections", error as Error); diff --git a/packages/worker/index.ts b/packages/worker/index.ts index 20da2f0..e461eee 100644 --- a/packages/worker/index.ts +++ b/packages/worker/index.ts @@ -1,9 +1,11 @@ export { WorkerPool } from "./pool"; export { processBatch } from "./process"; -export type { - EngineTarget, - ProcessResult, - WorkerConfig, - WorkerStats, +export { + DEFAULT_WORKER_TIMEOUTS, + type ProcessResult, + type SpaceTarget, + type WorkerConfig, + type WorkerStats, + type WorkerTimeouts, } from "./types"; export { Worker } from "./worker"; diff --git a/packages/worker/package.json b/packages/worker/package.json index 9adb437..7e661ce 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -5,8 +5,9 @@ "type": "module", "main": "index.ts", "dependencies": { - "@memory.build/engine": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", - "@pydantic/logfire-node": "^0.13.1" + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9" } } diff --git a/packages/worker/pool.ts b/packages/worker/pool.ts index 0d4a806..000340f 100644 --- a/packages/worker/pool.ts +++ b/packages/worker/pool.ts @@ -1,19 +1,19 @@ -import type { SQL } from "bun"; +import type { Sql } from "postgres"; import type { WorkerConfig, WorkerStats } from "./types"; import { Worker } from "./worker"; /** * Pool of N embedding workers using the provided SQL connection pool. - * Each worker independently discovers engines, shuffles its target list, + * Each worker independently discovers spaces, shuffles its target list, * and polls queues. FOR UPDATE SKIP LOCKED prevents double-processing. */ export class WorkerPool { - private readonly sql: SQL; + private readonly sql: Sql; private readonly config: WorkerConfig; private workers: Worker[] = []; private running = false; - constructor(sql: SQL, config: WorkerConfig) { + constructor(sql: Sql, config: WorkerConfig) { this.sql = sql; this.config = config; } @@ -44,7 +44,7 @@ export class WorkerPool { totalProcessed: 0, totalFailed: 0, totalPruned: 0, - enginesDropped: 0, + spacesDropped: 0, consecutiveErrors: 0, }; for (const worker of this.workers) { @@ -53,7 +53,7 @@ export class WorkerPool { agg.totalProcessed += s.totalProcessed; agg.totalFailed += s.totalFailed; agg.totalPruned += s.totalPruned; - agg.enginesDropped += s.enginesDropped; + agg.spacesDropped += s.spacesDropped; agg.consecutiveErrors = Math.max( agg.consecutiveErrors, s.consecutiveErrors, diff --git a/packages/worker/process.integration.test.ts b/packages/worker/process.integration.test.ts index 63712cf..8672457 100644 --- a/packages/worker/process.integration.test.ts +++ b/packages/worker/process.integration.test.ts @@ -1,270 +1,193 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { + bootstrapSpaceDatabase, + generateSlug, + migrateSpace, + slugToSchema, +} from "@memory.build/database"; import { RateLimitError } from "@memory.build/embedding"; -import { createEngineDB } from "@memory.build/engine/db"; -import { bootstrap } from "@memory.build/engine/migrate/bootstrap"; -import { provisionEngine } from "@memory.build/engine/migrate/provision"; -import { TestDatabase } from "@memory.build/engine/migrate/test-utils"; -import { SQL } from "bun"; +import postgres, { type Sql } from "postgres"; import { processBatch, pruneQueue } from "./process"; -import type { WorkerConfig } from "./types"; +import type { SpaceTarget, WorkerConfig } from "./types"; // --------------------------------------------------------------------------- -// Test setup +// Test setup — a real space schema (me_) on the new-model pool. // --------------------------------------------------------------------------- -const testDb = new TestDatabase(); -let connectionString: string; -let sql: SQL; -const slug = "tstworker001"; -const schema = `me_${slug}`; -const target = { schema, shard: 1 }; -const discover = async () => [target]; - -beforeAll(async () => { - connectionString = await testDb.create(); - sql = new SQL(connectionString); - await bootstrap(sql); - await provisionEngine(sql, slug, undefined, "0.1.0"); - - // Create a superuser principal for inserting memories - const db = createEngineDB(sql, schema); - const su = await db.createSuperuser("worker-test-admin"); - db.setUser(su.id); - - // Grant me_embed SELECT/UPDATE on memory (already done by migration 005) - // but we need to ensure the embedding_queue trigger is active -}); - -afterAll(async () => { - await sql.close(); - await testDb.drop(); -}); +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; -// --------------------------------------------------------------------------- -// Helper: insert a memory and return its id + queue state -// --------------------------------------------------------------------------- +let sql: Sql; +let slug: string; +let schema: string; +let target: SpaceTarget; +const discover = async () => [target]; -function getDb() { - return createEngineDB(sql, schema); +/** A config whose embedding calls hit the given mock base URL. */ +function mockConfig(baseUrl: string, maxRetries?: number): WorkerConfig { + return { + embedding: { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + apiKey: "test-key", + baseUrl, + ...(maxRetries !== undefined ? { options: { maxRetries } } : {}), + }, + discover, + batchSize: 10, + }; } -async function withDb() { - const db = getDb(); - const su = await db.getUserByName("worker-test-admin"); - db.setUser(su!.id); - return db; +/** A mock OpenAI embeddings server returning a fixed vector. */ +function embedServer(): ReturnType { + const embedding = Array.from({ length: 1536 }, (_, i) => i * 0.001); + return Bun.serve({ + port: 0, + fetch() { + return Response.json({ + object: "list", + data: [{ object: "embedding", embedding, index: 0 }], + model: "text-embedding-3-small", + usage: { prompt_tokens: 5, total_tokens: 5 }, + }); + }, + }); } async function insertMemory(content: string): Promise { - const db = await withDb(); - const memory = await db.createMemory({ content, tree: "test.worker" }); - return memory.id; + const [row] = await sql.unsafe( + `INSERT INTO ${schema}.memory (content, tree) VALUES ($1, ''::ltree) RETURNING id`, + [content], + ); + return row?.id as string; } -async function getQueueEntries(memoryId: string) { +function getQueueEntries(memoryId: string) { return sql.unsafe( `SELECT * FROM ${schema}.embedding_queue WHERE memory_id = $1 ORDER BY id`, [memoryId], - ); + ) as Promise[]>; } -async function getMemoryEmbedding(memoryId: string) { - const [row] = await sql.unsafe( - `SELECT embedding, embedding_version FROM ${schema}.memory WHERE id = $1`, - [memoryId], +async function clearPending() { + await sql.unsafe( + `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, ); - return row; } -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + slug = generateSlug(); + schema = slugToSchema(slug); + target = { schema }; + await bootstrapSpaceDatabase(sql); + await migrateSpace(sql, { slug }); +}); + +afterAll(async () => { + await sql.unsafe(`DROP SCHEMA IF EXISTS ${schema} CASCADE`); + await sql.end(); +}); + +describe("processBatch integration (space model)", () => { + beforeEach(clearPending); -describe("processBatch integration", () => { test("processes queue entries and writes embeddings", async () => { const memoryId = await insertMemory("Hello world embedding test"); - // Verify queue entry was created by trigger const queueBefore = await getQueueEntries(memoryId); expect(queueBefore.length).toBeGreaterThanOrEqual(1); - expect(queueBefore[0].outcome).toBeNull(); - - // We need to mock generateEmbeddings at the module level - // Instead, use a real-ish approach: create a processBatch wrapper - // that intercepts. For integration test, we'll use the actual processBatch - // but with a test embedding provider. - - // Create a mock embedding server using Bun.serve - const mockEmbedding = Array.from({ length: 1536 }, (_, i) => i * 0.001); - const server = Bun.serve({ - port: 0, - fetch() { - return Response.json({ - object: "list", - data: [{ object: "embedding", embedding: mockEmbedding, index: 0 }], - model: "text-embedding-3-small", - usage: { prompt_tokens: 5, total_tokens: 5 }, - }); - }, - }); + expect(queueBefore[0]?.outcome).toBeNull(); + const server = embedServer(); try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`), + ); expect(result.claimed).toBeGreaterThanOrEqual(1); expect(result.succeeded).toBeGreaterThanOrEqual(1); expect(result.failed).toBe(0); - // Verify embedding was written - const mem = await getMemoryEmbedding(memoryId); - expect(mem.embedding).toBeDefined(); + const [mem] = await sql.unsafe( + `SELECT embedding FROM ${schema}.memory WHERE id = $1`, + [memoryId], + ); + expect(mem?.embedding).toBeDefined(); - // Verify queue entry marked completed const queueAfter = await getQueueEntries(memoryId); - const completed = queueAfter.find( - (q: Record) => q.outcome === "completed", - ); - expect(completed).toBeDefined(); + expect(queueAfter.some((q) => q.outcome === "completed")).toBe(true); } finally { server.stop(); } }); - test("handles stale version (content changed during embed)", async () => { - await insertMemory("Original content for version test"); - - // Manually bump embedding_version to simulate content change after claim - // First, process to clear the initial queue entry - const server = Bun.serve({ - port: 0, - fetch() { - return Response.json({ - object: "list", - data: [ - { - object: "embedding", - embedding: Array.from({ length: 1536 }, (_, i) => i * 0.001), - index: 0, - }, - ], - model: "text-embedding-3-small", - usage: { prompt_tokens: 5, total_tokens: 5 }, - }); - }, - }); + test("cancels stale version at claim time", async () => { + const staleId = await insertMemory("Stale version content"); + // Bump the memory's version so the queued row (v1) is stale. + await sql.unsafe( + `UPDATE ${schema}.memory SET embedding_version = embedding_version + 1 WHERE id = $1`, + [staleId], + ); + const server = embedServer(); try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - // Clear any pending entries first - await processBatch(sql, target, config); - - // Now insert a new memory and manually create a stale queue entry - const staleId = await insertMemory("Stale version content"); - - // Bump the memory's embedding_version to make queue entry stale - await sql.unsafe( - `UPDATE ${schema}.memory SET embedding_version = embedding_version + 1 WHERE id = $1`, - [staleId], + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`), ); - - const result = await processBatch(sql, target, config); - - // Stale row cancelled at claim time — not counted as claimed expect(result.claimed).toBe(0); - - // Queue entry should be cancelled (version mismatch detected at claim) const queue = await getQueueEntries(staleId); - const cancelled = queue.find( - (q: Record) => q.outcome === "cancelled", - ); - expect(cancelled).toBeDefined(); + expect(queue.some((q) => q.outcome === "cancelled")).toBe(true); } finally { server.stop(); } }); - test("handles non-rate-limit embedding errors gracefully", async () => { + test("non-rate-limit error records last_error, leaves outcome NULL for retry", async () => { const memoryId = await insertMemory("Error test content"); - - // Mock server that returns a non-rate-limit error (400 bad request) const server = Bun.serve({ port: 0, fetch() { return new Response( JSON.stringify({ - error: { - message: "Invalid input", - type: "invalid_request_error", - }, + error: { message: "Invalid input", type: "invalid_request_error" }, }), { status: 400 }, ); }, }); - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - options: { maxRetries: 0 }, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`, 0), + ); expect(result.claimed).toBeGreaterThanOrEqual(1); expect(result.failed).toBeGreaterThanOrEqual(1); - // Queue entry should still have NULL outcome (for retry) but have last_error set const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === null && q.last_error, - ); + const entry = queue.find((q) => q.outcome === null && q.last_error); expect(entry).toBeDefined(); - expect(entry.last_error).toBeTruthy(); + expect(entry?.last_error).toBeTruthy(); } finally { server.stop(); } }); test("rate limit (429) throws RateLimitError and decrements attempts", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - const memoryId = await insertMemory("Rate limit test content"); - - // Mock server that returns 429 const server = Bun.serve({ port: 0, fetch() { @@ -272,141 +195,80 @@ describe("processBatch integration", () => { JSON.stringify({ error: { message: "Rate limited", type: "rate_limit_error" }, }), - { - status: 429, - headers: { "retry-after-ms": "5000" }, - }, + { status: 429, headers: { "retry-after-ms": "5000" } }, ); }, }); - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - options: { maxRetries: 0 }, - }, - discover, - batchSize: 10, - }; - - // processBatch should throw RateLimitError - await expect(processBatch(sql, target, config)).rejects.toBeInstanceOf( - RateLimitError, - ); - - // Queue entry should have attempts back to 0 (claim incremented to 1, - // then RateLimitError handler decremented back to 0) + await expect( + processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`, 0), + ), + ).rejects.toBeInstanceOf(RateLimitError); + + // claim incremented attempts to 1, the RateLimitError handler decremented + // it back to 0 so the transient failure isn't charged. const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === null, - ); - expect(entry).toBeDefined(); - expect(entry.attempts).toBe(0); + const entry = queue.find((q) => q.outcome === null); + expect(entry?.attempts).toBe(0); } finally { server.stop(); } }); - test("marks queue row as failed after max attempts exhausted (non-rate-limit)", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - - const memoryId = await insertMemory("Exhaust attempts content"); - - // Set attempts = 2 so next claim brings it to 3 (== max_attempts) + test("claim sweeps exhausted rows to 'failed'", async () => { + const memoryId = await insertMemory("Exhausted attempts content"); + // Simulate a crash that left the row at max attempts with an expired lock. await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET attempts = 2 WHERE memory_id = $1 AND outcome IS NULL`, + `UPDATE ${schema}.embedding_queue + SET attempts = 3, vt = now() - interval '1 minute' + WHERE memory_id = $1 AND outcome IS NULL`, [memoryId], ); - // Mock server that returns 400 (non-rate-limit) so embedding fails - const server = Bun.serve({ - port: 0, - fetch() { - return new Response( - JSON.stringify({ - error: { - message: "Invalid input", - type: "invalid_request_error", - }, - }), - { status: 400 }, - ); - }, - }); - - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - options: { maxRetries: 0 }, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - - expect(result.claimed).toBeGreaterThanOrEqual(1); - expect(result.failed).toBeGreaterThanOrEqual(1); - - // Queue entry should now be finalized as 'failed' - const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === "failed", - ); - expect(entry).toBeDefined(); - expect(entry.last_error).toBeTruthy(); - } finally { - server.stop(); - } - }); - - test("cancels stale queue rows at claim time", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, + // Base URL is never reached — the row is swept, not embedded. + const result = await processBatch( + sql, + target, + mockConfig("http://localhost:1/v1"), ); + expect(result.claimed).toBe(0); - // Insert a memory — trigger creates queue row at version 1 - const memoryId = await insertMemory("Stale claim-time v1"); - - // Update content twice more — each triggers a new queue row (v2, v3) - const db = await withDb(); - await db.updateMemory(memoryId, { content: "Stale claim-time v2" }); - await db.updateMemory(memoryId, { content: "Stale claim-time v3" }); + const queue = await getQueueEntries(memoryId); + const entry = queue.find((q) => q.outcome === "failed"); + expect(entry).toBeDefined(); + expect(String(entry?.last_error)).toContain("exceeded max attempts"); + }); - // Verify 3 pending queue rows exist - const queueBefore = await getQueueEntries(memoryId); - const pending = queueBefore.filter( - (q: Record) => q.outcome === null, + test("cancels superseded versions, embeds only the latest", async () => { + const memoryId = await insertMemory("claim-time v1"); + await sql.unsafe(`UPDATE ${schema}.memory SET content = $1 WHERE id = $2`, [ + "claim-time v2", + memoryId, + ]); + await sql.unsafe(`UPDATE ${schema}.memory SET content = $1 WHERE id = $2`, [ + "claim-time v3", + memoryId, + ]); + + const pending = (await getQueueEntries(memoryId)).filter( + (q) => q.outcome === null, ); expect(pending.length).toBe(3); - // Mock server tracks call count — only version 3 should be embedded - let embedCallCount = 0; - const mockEmbedding = Array.from({ length: 1536 }, () => 0); + let embedCalls = 0; const server = Bun.serve({ port: 0, fetch() { - embedCallCount++; + embedCalls++; return Response.json({ object: "list", data: [ { object: "embedding", - embedding: mockEmbedding, + embedding: Array.from({ length: 1536 }, () => 0), index: 0, }, ], @@ -415,173 +277,54 @@ describe("processBatch integration", () => { }); }, }); - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - - // Only version 3 should be claimed and processed + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`), + ); expect(result.claimed).toBe(1); expect(result.succeeded).toBe(1); - expect(embedCallCount).toBe(1); + expect(embedCalls).toBe(1); - // Verify queue outcomes: two cancelled (v1, v2), one completed (v3) const queueAfter = await getQueueEntries(memoryId); - const cancelled = queueAfter.filter( - (q: Record) => q.outcome === "cancelled", + expect(queueAfter.filter((q) => q.outcome === "cancelled").length).toBe( + 2, ); - const completed = queueAfter.filter( - (q: Record) => q.outcome === "completed", + expect(queueAfter.filter((q) => q.outcome === "completed").length).toBe( + 1, ); - expect(cancelled.length).toBe(2); - expect(completed.length).toBe(1); } finally { server.stop(); } }); - test("cancels queue rows for deleted memories at claim time", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - - // Insert a memory — trigger creates a queue entry + test("deleted memory: queue row CASCADE-deleted, nothing to claim", async () => { const memoryId = await insertMemory("Delete claim-time test"); - - // Verify queue entry exists - const queueBefore = await getQueueEntries(memoryId); - expect(queueBefore.length).toBeGreaterThanOrEqual(1); - - // Delete the memory — CASCADE deletes the queue row too await sql.unsafe(`DELETE FROM ${schema}.memory WHERE id = $1`, [memoryId]); + expect((await getQueueEntries(memoryId)).length).toBe(0); - // Queue row should be gone due to CASCADE - const queueAfter = await getQueueEntries(memoryId); - expect(queueAfter.length).toBe(0); - - // processBatch should handle this gracefully (nothing to claim) - const server = Bun.serve({ - port: 0, - fetch() { - return Response.json({ - object: "list", - data: [ - { - object: "embedding", - embedding: Array.from({ length: 1536 }, () => 0), - index: 0, - }, - ], - model: "text-embedding-3-small", - usage: { prompt_tokens: 1, total_tokens: 1 }, - }); - }, - }); - - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - expect(result.claimed).toBe(0); - } finally { - server.stop(); - } - }); - - test("sweeps zombie rows as failed when attempts exhausted by crash", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - - const memoryId = await insertMemory("Zombie crash test content"); - - // Simulate a crash: set attempts = max_attempts and vt in the past, leave outcome NULL - await sql.unsafe( - `UPDATE ${schema}.embedding_queue - SET attempts = max_attempts, vt = now() - interval '1 minute' - WHERE memory_id = $1 AND outcome IS NULL`, - [memoryId], + const result = await processBatch( + sql, + target, + mockConfig("http://localhost:1/v1"), ); - - // processBatch should sweep the zombie row as failed, not claim it - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: "http://localhost:1/v1", // never called - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - - // Zombie was swept, not claimed expect(result.claimed).toBe(0); - - // Queue entry should be finalized as 'failed' with crash message - const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === "failed", - ); - expect(entry).toBeDefined(); - expect(entry.last_error).toContain("exceeded max attempts (worker crash)"); }); test("pruneQueue deletes terminal rows older than retention", async () => { - // Clear queue await sql.unsafe(`DELETE FROM ${schema}.embedding_queue`); - const memoryId = await insertMemory("Prune helper test memory"); - // Discard the trigger-enqueued row so we control all rows in the queue await sql.unsafe(`DELETE FROM ${schema}.embedding_queue`); - // Old terminal rows (will be pruned) await sql.unsafe( `INSERT INTO ${schema}.embedding_queue (memory_id, embedding_version, outcome, created_at) VALUES ($1, 1, 'completed', now() - interval '10 days'), ($1, 2, 'failed', now() - interval '10 days'), - ($1, 3, 'cancelled', now() - interval '10 days')`, - [memoryId], - ); - // Recent terminal row (kept) - await sql.unsafe( - `INSERT INTO ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - VALUES ($1, 4, 'completed', now() - interval '1 day')`, - [memoryId], - ); - // Old active row (kept — outcome IS NULL) - await sql.unsafe( - `INSERT INTO ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - VALUES ($1, 5, null, now() - interval '30 days')`, + ($1, 3, 'cancelled', now() - interval '10 days'), + ($1, 4, 'completed', now() - interval '1 day'), + ($1, 5, null, now() - interval '30 days')`, [memoryId], ); @@ -594,40 +337,22 @@ describe("processBatch integration", () => { [memoryId], )) as { embedding_version: number; outcome: string | null }[]; expect(remaining).toHaveLength(2); - expect(remaining[0]!.embedding_version).toBe(4); - expect(remaining[0]!.outcome).toBe("completed"); - expect(remaining[1]!.embedding_version).toBe(5); - expect(remaining[1]!.outcome).toBeNull(); + expect(remaining[0]?.embedding_version).toBe(4); + expect(remaining[1]?.embedding_version).toBe(5); + expect(remaining[1]?.outcome).toBeNull(); }); test("pruneQueue is a no-op when nothing matches", async () => { await sql.unsafe(`DELETE FROM ${schema}.embedding_queue`); - const pruned = await pruneQueue(sql, target, "7 days"); - expect(pruned).toBe(0); + expect(await pruneQueue(sql, target, "7 days")).toBe(0); }); test("returns zero when queue is empty", async () => { - // Use a dedicated config pointing at a mock server that should never be called - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: "http://localhost:1/v1", - }, - discover, - batchSize: 10, - }; - - // Clear all pending queue entries first - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, + const result = await processBatch( + sql, + target, + mockConfig("http://localhost:1/v1"), ); - - const result = await processBatch(sql, target, config); - expect(result.claimed).toBe(0); - expect(result.succeeded).toBe(0); - expect(result.failed).toBe(0); + expect(result).toEqual({ claimed: 0, succeeded: 0, failed: 0 }); }); }); diff --git a/packages/worker/process.ts b/packages/worker/process.ts index 93ce22b..301671d 100644 --- a/packages/worker/process.ts +++ b/packages/worker/process.ts @@ -3,20 +3,21 @@ import { generateEmbeddings, RateLimitError, } from "@memory.build/embedding"; -import { - DEFAULT_ENGINE_TIMEOUTS, - type EngineTimeouts, - setLocalEngineTimeouts, -} from "@memory.build/engine/ops/_tx"; import { info, reportError, span, warning } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import type { EngineTarget, ProcessResult, WorkerConfig } from "./types"; +import type { Sql } from "postgres"; +import { + DEFAULT_WORKER_TIMEOUTS, + type ProcessResult, + type SpaceTarget, + type WorkerConfig, + type WorkerTimeouts, +} from "./types"; -function workerEngineTimeouts(config?: WorkerConfig): EngineTimeouts { - return config?.workerEngineTimeouts ?? DEFAULT_ENGINE_TIMEOUTS; +function workerTimeouts(config?: WorkerConfig): WorkerTimeouts { + return config?.timeouts ?? DEFAULT_WORKER_TIMEOUTS; } -function workerEngineTimeoutAttributes(timeouts: EngineTimeouts) { +function timeoutAttributes(timeouts: WorkerTimeouts) { return { "db.statement_timeout": timeouts.statementTimeout, "db.lock_timeout": timeouts.lockTimeout, @@ -26,6 +27,32 @@ function workerEngineTimeoutAttributes(timeouts: EngineTimeouts) { }; } +/** + * Set transaction-local search_path + timeouts. The new model is not sharded + * and the space functions are security-invoker, so the worker runs as the pool + * user with no SET ROLE. + */ +async function prepareTx( + tx: Sql, + schema: string, + timeouts: WorkerTimeouts, +): Promise { + await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); + await tx.unsafe("SELECT set_config('statement_timeout', $1, true)", [ + timeouts.statementTimeout, + ]); + await tx.unsafe("SELECT set_config('lock_timeout', $1, true)", [ + timeouts.lockTimeout, + ]); + await tx.unsafe("SELECT set_config('transaction_timeout', $1, true)", [ + timeouts.transactionTimeout, + ]); + await tx.unsafe( + "SELECT set_config('idle_in_transaction_session_timeout', $1, true)", + [timeouts.idleInTransactionSessionTimeout], + ); +} + function asError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } @@ -37,24 +64,21 @@ function asError(error: unknown): Error { * Returns the number of rows pruned. */ export async function pruneQueue( - sql: SQL, - target: EngineTarget, + sql: Sql, + target: SpaceTarget, retention: string, config?: WorkerConfig, ): Promise { - const { schema, shard } = target; - const timeouts = workerEngineTimeouts(config); + const { schema } = target; + const timeouts = workerTimeouts(config); return sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + await prepareTx(tx as unknown as Sql, schema, timeouts); const rows = (await tx.unsafe( `SELECT ${schema}.prune_embedding_queue($1::interval) AS pruned`, [retention], )) as { pruned: string | number | null }[]; return Number(rows[0]?.pruned ?? 0); - }); + }) as Promise; } interface ClaimedRow { @@ -69,30 +93,30 @@ interface ClaimedRow { * * Claim and write-back are separate transactions — if the worker crashes * between them, the visibility timeout expires and rows become claimable again. + * Rows that exhaust their attempts are finalized to 'failed' by the claim + * function's sweep, so write-back only records last_error and leaves the + * outcome NULL on transient failure. */ export async function processBatch( - sql: SQL, - target: EngineTarget, + sql: Sql, + target: SpaceTarget, config: WorkerConfig, ): Promise { - const { schema, shard } = target; + const { schema } = target; const batchSize = config.batchSize ?? 10; const lockDuration = config.lockDuration ?? "5 minutes"; - const timeouts = workerEngineTimeouts(config); - const timeoutAttributes = workerEngineTimeoutAttributes(timeouts); + const timeouts = workerTimeouts(config); + const attrs = timeoutAttributes(timeouts); // --- Claim --- const claimStart = performance.now(); - const claimed = await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + const claimed = (await sql.begin(async (tx) => { + await prepareTx(tx as unknown as Sql, schema, timeouts); return tx.unsafe( `SELECT * FROM ${schema}.claim_embedding_batch($1, $2::interval)`, [batchSize, lockDuration], - ) as Promise; - }); + ); + })) as ClaimedRow[]; const claimDurationMs = performance.now() - claimStart; if (claimed.length === 0) { @@ -101,28 +125,25 @@ export async function processBatch( info("Embedding batch claimed", { "worker.schema": schema, - "worker.shard": shard, "batch.claimed": claimed.length, "batch.requested_size": batchSize, "batch.lock_duration": lockDuration, "batch.claim_duration_ms": claimDurationMs, "batch.memoryIds": claimed.map((r) => r.memory_id), "batch.queueIds": claimed.map((r) => r.queue_id), - ...timeoutAttributes, + ...attrs, }); - // Process claimed items with telemetry return span("embedding.batch", { attributes: { "worker.schema": schema, - "worker.shard": shard, "batch.size": claimed.length, "batch.requested_size": batchSize, "batch.lock_duration": lockDuration, "batch.claim_duration_ms": claimDurationMs, "batch.memoryIds": claimed.map((r) => r.memory_id), "batch.queueIds": claimed.map((r) => r.queue_id), - ...timeoutAttributes, + ...attrs, }, callback: async () => { // --- Embed --- @@ -137,12 +158,9 @@ export async function processBatch( } catch (error) { if (error instanceof RateLimitError) { // Undo the attempt increment from claim — rate limits are transient - // and should not consume max_attempts + // and should not consume the attempt budget. await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + await prepareTx(tx as unknown as Sql, schema, timeouts); for (const row of claimed) { await tx.unsafe( `UPDATE ${schema}.embedding_queue @@ -158,14 +176,12 @@ export async function processBatch( info("Embedding batch generated", { "worker.schema": schema, - "worker.shard": shard, "batch.claimed": claimed.length, "batch.generated": embedResults.length, "batch.embed_successes": embedResults.filter((r) => !r.error).length, "batch.embed_errors": embedResults.filter((r) => r.error).length, }); - // Build lookup: memory_id → embed result const resultMap = new Map(embedResults.map((r) => [r.id, r])); // --- Write-back --- @@ -175,32 +191,29 @@ export async function processBatch( await span("embedding.write_back", { attributes: { "worker.schema": schema, - "worker.shard": shard, "batch.size": claimed.length, "batch.embed_successes": embedResults.filter((r) => !r.error).length, "batch.embed_errors": embedResults.filter((r) => r.error).length, - ...timeoutAttributes, + ...attrs, }, callback: async () => { for (const row of claimed) { try { await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + await prepareTx(tx as unknown as Sql, schema, timeouts); const result = resultMap.get(row.memory_id); if (!result || result.error) { - // Embedding failed — record error, leave outcome NULL for retry. - // Queue row may be CASCADE-deleted if memory was deleted; 0 rows is fine. + // Embedding failed — record the error, leave outcome NULL so + // the row retries; the claim sweep fails it once attempts are + // exhausted. (Row may be CASCADE-deleted if the memory was + // deleted; 0 rows updated is fine.) const error = result?.error ?? "No embedding result returned"; await tx.unsafe( `UPDATE ${schema}.embedding_queue SET last_error = $1 - , outcome = CASE WHEN attempts >= max_attempts THEN 'failed' END - WHERE id = $2`, + WHERE id = $2 AND outcome IS NULL`, [error, row.queue_id], ); failed++; @@ -218,8 +231,7 @@ export async function processBatch( ); if (updated.length === 0) { - // Content changed or memory deleted between claim and embed — cancel. - // Queue row may already be CASCADE-deleted; 0 rows updated is fine. + // Content changed or memory deleted between claim and embed. await tx.unsafe( `UPDATE ${schema}.embedding_queue SET outcome = 'cancelled' @@ -227,9 +239,6 @@ export async function processBatch( [row.queue_id], ); } else { - // Embedding written — mark completed. - // Queue row may be CASCADE-deleted if memory deleted between these two - // statements; 0 rows updated is fine. await tx.unsafe( `UPDATE ${schema}.embedding_queue SET outcome = 'completed' @@ -244,7 +253,6 @@ export async function processBatch( failed++; reportError("Embedding row write-back failed", err, { "worker.schema": schema, - "worker.shard": shard, "queue.id": row.queue_id, "memory.id": row.memory_id, "memory.embedding_version": row.embedding_version, @@ -252,14 +260,10 @@ export async function processBatch( try { await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + await prepareTx(tx as unknown as Sql, schema, timeouts); await tx.unsafe( `UPDATE ${schema}.embedding_queue SET last_error = $1 - , outcome = CASE WHEN attempts >= max_attempts THEN 'failed' END WHERE id = $2 AND outcome IS NULL`, [err.message, row.queue_id], ); @@ -267,7 +271,6 @@ export async function processBatch( } catch (recordError) { warning("Failed to record embedding row write-back error", { "worker.schema": schema, - "worker.shard": shard, "queue.id": row.queue_id, "memory.id": row.memory_id, error: asError(recordError).message, diff --git a/packages/worker/types.ts b/packages/worker/types.ts index 1388cbf..ee423cc 100644 --- a/packages/worker/types.ts +++ b/packages/worker/types.ts @@ -1,15 +1,29 @@ import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { EngineTimeouts } from "@memory.build/engine/ops/_tx"; -export interface EngineTarget { +/** A space schema (me_) the worker should process. */ +export interface SpaceTarget { schema: string; - shard: number; } +/** Transaction-local timeouts applied to each worker DB transaction. */ +export interface WorkerTimeouts { + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export const DEFAULT_WORKER_TIMEOUTS: WorkerTimeouts = { + statementTimeout: "25s", + lockTimeout: "5s", + transactionTimeout: "30s", + idleInTransactionSessionTimeout: "30s", +}; + export interface WorkerConfig { embedding: EmbeddingConfig; - /** Discover active engines (schema + shard) from accounts DB */ - discover: () => Promise; + /** Discover the spaces (me_ schemas) to process. */ + discover: () => Promise; /** Number of queue entries to claim per batch (default: 10) */ batchSize?: number; /** PostgreSQL interval for claim lock duration (default: '5 minutes') */ @@ -18,10 +32,10 @@ export interface WorkerConfig { idleDelayMs?: number; /** Maximum backoff delay on consecutive errors (default: 60_000ms) */ maxBackoffMs?: number; - /** How often to re-discover engines (default: 60_000ms) */ + /** How often to re-discover spaces (default: 60_000ms) */ refreshIntervalMs?: number; - /** PostgreSQL transaction/session timeouts for worker engine DB work */ - workerEngineTimeouts?: EngineTimeouts; + /** PostgreSQL transaction/session timeouts for worker DB work */ + timeouts?: WorkerTimeouts; /** Exit gracefully after this much idle time (optional) */ drainTimeoutMs?: number; /** @@ -43,11 +57,11 @@ export interface WorkerStats { totalFailed: number; totalPruned: number; /** - * Number of times an engine was dropped from the in-memory target list - * because its schema no longer exists in PostgreSQL (e.g. engine deleted + * Number of times a space was dropped from the in-memory target list + * because its schema no longer exists in PostgreSQL (e.g. space deleted * between discover() refreshes). Self-heals on the next refresh. */ - enginesDropped: number; + spacesDropped: number; consecutiveErrors: number; lastError?: string; } diff --git a/packages/worker/worker.test.ts b/packages/worker/worker.test.ts index ebd8de2..22999f4 100644 --- a/packages/worker/worker.test.ts +++ b/packages/worker/worker.test.ts @@ -1,6 +1,5 @@ import { describe, expect, test } from "bun:test"; import { RateLimitError } from "@memory.build/embedding"; -import { SQL } from "bun"; import { WorkerPool } from "./pool"; import { Worker } from "./worker"; @@ -76,7 +75,7 @@ describe("Worker", () => { }, discover: async () => { discoverCalls++; - return [{ schema: "me_test12345678", shard: 1 }]; + return [{ schema: "me_test12345678" }]; }, idleDelayMs: 50, drainTimeoutMs: 100, @@ -144,7 +143,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -204,7 +203,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -243,7 +242,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -281,9 +280,10 @@ describe("Worker", () => { } if (q.includes("claim_embedding_batch")) { if (lastSchema === deadSchema) { - throw new SQL.PostgresError( - `schema "${deadSchema}" does not exist`, - { code: "3F000", errno: "3F000" }, + // postgres.js surfaces SQLSTATE on `.code`; 3F000 = missing schema + throw Object.assign( + new Error(`schema "${deadSchema}" does not exist`), + { code: "3F000" }, ); } claimedBy.push(lastSchema ?? ""); @@ -302,10 +302,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [ - { schema: liveSchema, shard: 1 }, - { schema: deadSchema, shard: 1 }, - ], + discover: async () => [{ schema: liveSchema }, { schema: deadSchema }], idleDelayMs: 50, drainTimeoutMs: 200, refreshIntervalMs: 1_000_000, // don't refresh during test @@ -319,9 +316,9 @@ describe("Worker", () => { expect(worker.stats.consecutiveErrors).toBe(0); expect(worker.stats.lastError).toBeUndefined(); // Dead schema dropped exactly once, then filtered out of the rotation. - expect(worker.stats.enginesDropped).toBe(1); + expect(worker.stats.spacesDropped).toBe(1); // Live schema kept being polled across multiple loop iterations even - // though we only saw one enginesDropped — proves the live target wasn't + // though we only saw one spacesDropped — proves the live target wasn't // collateral damage when its sibling threw. expect(claimedBy.length).toBeGreaterThanOrEqual(2); expect(claimedBy.every((s) => s === liveSchema)).toBe(true); @@ -390,7 +387,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 200, refreshIntervalMs: 1_000_000, @@ -433,7 +430,7 @@ describe("WorkerPool", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -540,7 +537,7 @@ describe("WorkerPool", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 60_000, maxBackoffMs: 60_000, refreshIntervalMs: 1_000_000, diff --git a/packages/worker/worker.ts b/packages/worker/worker.ts index 055d45c..530098b 100644 --- a/packages/worker/worker.ts +++ b/packages/worker/worker.ts @@ -1,6 +1,6 @@ import { RateLimitError } from "@memory.build/embedding"; import { info, reportError, warning } from "@pydantic/logfire-node"; -import { SQL } from "bun"; +import type { Sql } from "postgres"; import { processBatch, pruneQueue } from "./process"; import type { WorkerConfig, WorkerStats } from "./types"; @@ -8,12 +8,12 @@ import type { WorkerConfig, WorkerStats } from "./types"; const RATE_LIMIT_FLOOR_MS = 30_000; /** - * SQLSTATE 3F000 = invalid_schema_name. Raised when the engine's schema - * no longer exists — typically because the engine was deleted between - * discover() refreshes. Treated as benign: drop the target and continue. + * SQLSTATE 3F000 = invalid_schema_name. Raised when the space's schema no + * longer exists — typically because the space was deleted between discover() + * refreshes. Treated as benign: drop the target and continue. */ function isMissingSchemaError(error: unknown): boolean { - return error instanceof SQL.PostgresError && error.errno === "3F000"; + return (error as { code?: string }).code === "3F000"; } /** @@ -51,14 +51,14 @@ function shuffle(arr: T[]): T[] { } /** - * Embedding worker. Discovers engines from the accounts DB and polls their - * embedding queues in round-robin, generating embeddings for new memories. + * Embedding worker. Discovers spaces and polls their embedding queues in + * round-robin, generating embeddings for new memories. * * Adaptive delay: loops immediately when work is found, sleeps idleDelayMs * when idle. Exponential backoff on consecutive errors. */ export class Worker { - private readonly sql: SQL; + private readonly sql: Sql; private readonly config: WorkerConfig; private abort: AbortController | null = null; private runPromise: Promise | null = null; @@ -67,11 +67,11 @@ export class Worker { totalProcessed: 0, totalFailed: 0, totalPruned: 0, - enginesDropped: 0, + spacesDropped: 0, consecutiveErrors: 0, }; - constructor(sql: SQL, config: WorkerConfig) { + constructor(sql: Sql, config: WorkerConfig) { this.sql = sql; this.config = config; } @@ -105,7 +105,7 @@ export class Worker { } async function run( - sql: SQL, + sql: Sql, config: WorkerConfig, signal: AbortSignal, stats: WorkerStats, @@ -153,10 +153,9 @@ async function run( if (isMissingSchemaError(err)) { warning("Embedding target schema no longer exists, dropping", { "worker.schema": target.schema, - "worker.shard": target.shard, }); droppedSchemas.add(target.schema); - stats.enginesDropped++; + stats.spacesDropped++; continue; } throw err; @@ -181,11 +180,10 @@ async function run( // Schema dropped between claim and prune in the same cycle. // Drop the target now to avoid re-trying it next cycle. droppedSchemas.add(target.schema); - stats.enginesDropped++; + stats.spacesDropped++; } else { warning("Embedding queue prune failed", { "worker.schema": target.schema, - "worker.shard": target.shard, error: pruneError instanceof Error ? pruneError.message @@ -229,7 +227,7 @@ async function run( warning("Rate limited by embedding provider, backing off", { backoffMs, retryAfterMs: error.retryAfterMs, - engineCount: targets.length, + spaceCount: targets.length, }); if (signal.aborted) break; @@ -243,7 +241,7 @@ async function run( stats.lastError = errorMsg; reportError("Worker batch processing failed", error as Error, { consecutiveErrors, - engineCount: targets.length, + spaceCount: targets.length, }); if (signal.aborted) break; @@ -258,7 +256,7 @@ async function run( } finally { info("Embedding worker stopped", { consecutiveErrors, - engineCount: targets.length, + spaceCount: targets.length, }); } } From 04e35de426381ca0efb06059286fca4a00082a56 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 11:29:41 +0200 Subject: [PATCH 045/156] docs(todo): worker write-back should call space SQL functions Note the follow-up to move process.ts's raw embedding_queue/memory write-back UPDATEs behind space SQL functions (complete/fail/release_embedding), matching the "logic in DB functions" principle the rest of the cutover follows. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TODO.md b/TODO.md index a874e96..bfc3bef 100644 --- a/TODO.md +++ b/TODO.md @@ -3,6 +3,22 @@ Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). +## Worker: call space SQL functions instead of raw queries + +The embedding worker's write-back in `packages/worker/process.ts` still issues +raw `UPDATE embedding_queue …` / `UPDATE memory SET embedding …` statements, +against the "logic in DB functions, TS calls functions" principle the rest of +the cutover follows. + +- [ ] Add space SQL functions for the write-back path (e.g. + `complete_embedding(queue_id, memory_id, embedding_version, embedding)` + that does the version-guarded memory update + sets the queue outcome to + `completed`/`cancelled` atomically, plus `fail_embedding(queue_id, error)` + and the rate-limit `release_embedding(queue_id)` attempt-undo), and have + `process.ts` call those instead of inline SQL. Claim already goes through + `claim_embedding_batch`; this finishes the job for the write-back/prune + side so the worker holds no embedded SQL. + ## Decision: `core` and `space` are one package (`@memory.build/database`) Resolved (2026-06): merged `packages/core` + `packages/space` into a single From e241e032db5db953d4b18a7e09910c96b90cb43a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 12:00:15 +0200 Subject: [PATCH 046/156] feat(core): space.list on the user endpoint + transitive space admin (4E-S1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give a logged-in user a way to discover the spaces they belong to (so the CLI can pick the X-Me-Space the rest of its commands are scoped to), and make space admin transfer through groups consistently with the rest of the Model 2 model. - core.list_spaces_for_member(_member_id): spaces the member belongs to — directly (principal_space) or via a group — with the admin flag. Driven from the membership tables (indexed by member) and PK-joined to space, not a scan of every space. Add a group_member(member_id, space_id) index for the no-space_id member lookup. - is_principal_space_admin is now transitive: a principal is a space admin via a direct admin membership OR by belonging to a group whose own space-membership is admin (agents still excluded). list_spaces_for_member derives its admin flag from it, so space.list and requireSpaceAdmin agree. - coreStore.listSpacesForMember + MemberSpace type; space.list on the user RPC (protocol user/space.ts + handler). - tests: transitive admin at the core level; space.list covering direct-admin, group-transitive-admin, group-only (non-admin), and none. - TODO notes: reconsider user-owned api keys; space invitations subsystem. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 24 +++++++++ .../idempotent/001_principal_space.sql | 36 ++++++++++--- .../core/migrate/idempotent/004_space.sql | 48 +++++++++++++++++ .../migrate/incremental/004_group_member.sql | 4 ++ packages/engine/core/core.integration.test.ts | 17 ++++++ packages/engine/core/db.ts | 12 +++++ packages/engine/core/index.ts | 1 + packages/engine/core/types.ts | 5 ++ packages/protocol/package.json | 2 +- packages/protocol/user/index.ts | 6 ++- packages/protocol/user/space.ts | 28 ++++++++++ .../server/rpc/user/agent.integration.test.ts | 53 +++++++++++++++++++ packages/server/rpc/user/index.ts | 8 ++- packages/server/rpc/user/space.ts | 42 +++++++++++++++ 14 files changed, 276 insertions(+), 10 deletions(-) create mode 100644 packages/protocol/user/space.ts create mode 100644 packages/server/rpc/user/space.ts diff --git a/TODO.md b/TODO.md index bfc3bef..b410514 100644 --- a/TODO.md +++ b/TODO.md @@ -3,6 +3,30 @@ Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). +## Reconsider: api keys for users (not just agents) + +Keys are currently agent-only (`apiKey.create` is gated by `requireOwnedAgent`; +humans authenticate via session). The intended CLI surface treats `ME_API_KEY` +as pointing to a "user|agent" and `me apikey create` defaulting to self, which +implies users can mint their own keys. + +- [ ] Decide whether to allow user-owned api keys. `validate_api_key` already + returns the principal regardless of kind, and `authenticateSpace` would + work unchanged — so it's mostly relaxing the `apiKey.create` gate to allow + `member == self` (a user) in addition to agents the caller owns. Weigh + against the "humans use sessions only" security stance. + +## Space invitations + +The CLI spec includes `me space invite` / `invite list` / `invite revoke` +(invite a user by email into a space with an initial role/grant). Deferred from +4E — it's a new subsystem. + +- [ ] Design + build space-scoped invitations: a core table (space_id, email, + role/grant, token, status, expiry), RPC on the space endpoint + (invite.create/list/revoke + accept on the user endpoint), and the email/ + link delivery. Mirrors the device-flow consent UX where relevant. + ## Worker: call space SQL functions instead of raw queries The embedding worker's write-back in `packages/worker/process.ts` still issues diff --git a/packages/database/core/migrate/idempotent/001_principal_space.sql b/packages/database/core/migrate/idempotent/001_principal_space.sql index e939a6d..a112510 100644 --- a/packages/database/core/migrate/idempotent/001_principal_space.sql +++ b/packages/database/core/migrate/idempotent/001_principal_space.sql @@ -19,6 +19,10 @@ $func$ language sql stable security invoker ------------------------------------------------------------------------------- -- is_principal_space_admin +-- A principal is a space admin if it has a direct admin membership, OR it is a +-- member of a group whose own space-membership is admin (admin transfers +-- transitively through groups, like access does — Model 2). Agents are never +-- space admins. ------------------------------------------------------------------------------- create or replace function {{schema}}.is_principal_space_admin ( _principal_id uuid @@ -26,16 +30,34 @@ create or replace function {{schema}}.is_principal_space_admin ) returns bool as $func$ - select coalesce + select exists ( + select 1 + from {{schema}}.principal p + where p.id = _principal_id + and p.kind <> 'a' -- agents cannot be space admins + and ( - select ps.admin and (not p.kind = 'a') -- agents cannot be space admins - from {{schema}}.principal_space ps - inner join {{schema}}.principal p on (ps.principal_id = p.id) - where ps.principal_id = _principal_id - and ps.space_id = _space_id + -- direct admin membership + exists + ( + select 1 + from {{schema}}.principal_space ps + where ps.principal_id = p.id + and ps.space_id = _space_id + and ps.admin + ) + -- admin inherited from an admin group the principal belongs to + or exists + ( + select 1 + from {{schema}}.group_member gm + inner join {{schema}}.principal_space gps + on (gps.principal_id = gm.group_id and gps.space_id = _space_id and gps.admin) + where gm.member_id = p.id + and gm.space_id = _space_id + ) ) - , false ) $func$ language sql stable security invoker ; diff --git a/packages/database/core/migrate/idempotent/004_space.sql b/packages/database/core/migrate/idempotent/004_space.sql index cb31721..c7f6bac 100644 --- a/packages/database/core/migrate/idempotent/004_space.sql +++ b/packages/database/core/migrate/idempotent/004_space.sql @@ -58,3 +58,51 @@ as $func$ $func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; + +------------------------------------------------------------------------------- +-- list_spaces_for_member +-- Spaces a member (user/agent) belongs to — directly (principal_space) or +-- through a group (Model 2). `admin` is the direct-membership admin flag. +-- Used by the user endpoint so a logged-in human can pick their space. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_spaces_for_member +( _member_id uuid +) +returns table +( id uuid +, slug text +, name text +, language text +, admin bool +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + -- Drive from the membership tables (indexed by the member) and PK-join to + -- space, rather than scanning every space and probing membership per row. + with space_ids as + ( + select ps.space_id + from {{schema}}.principal_space ps + where ps.principal_id = _member_id + union + select gm.space_id + from {{schema}}.group_member gm + where gm.member_id = _member_id + ) + select + s.id + , s.slug + , s.name::text + , s.language + -- derived from is_principal_space_admin so it matches the authority gate + -- (includes admin inherited via an admin group) + , {{schema}}.is_principal_space_admin(_member_id, s.id) as admin + , s.created_at + , s.updated_at + from {{schema}}.space s + inner join space_ids si on si.space_id = s.id + order by s.created_at desc +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/incremental/004_group_member.sql b/packages/database/core/migrate/incremental/004_group_member.sql index e29cc50..c041afc 100644 --- a/packages/database/core/migrate/incremental/004_group_member.sql +++ b/packages/database/core/migrate/incremental/004_group_member.sql @@ -13,3 +13,7 @@ create table {{schema}}.group_member -- index for listing groups in a space and/or members of a group create index on {{schema}}.group_member (space_id, group_id, member_id) include (admin); + +-- index for finding a member's spaces/groups across the whole space set +-- (e.g. list_spaces_for_member's member_id lookup, with no space_id filter) +create index on {{schema}}.group_member (member_id, space_id); diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts index 7dc6e95..f3d9ed2 100644 --- a/packages/engine/core/core.integration.test.ts +++ b/packages/engine/core/core.integration.test.ts @@ -130,6 +130,23 @@ test("groups: create, list, rename, members, delete", async () => { expect(await core.listSpaceGroups(spaceId)).toHaveLength(0); }); +test("space admin transfers through an admin group", async () => { + const groupId = await core.createGroup(spaceId, `admins_${rand(6)}`); + // designate the group itself as an admin member of the space + await core.addPrincipalToSpace(spaceId, groupId, true); + + // a user added only to that group inherits space-admin transitively + const member = await v7(); + await core.createUser(member, `m_${rand(8)}@example.com`); + await core.addGroupMember(spaceId, groupId, member); + expect(await core.isSpaceAdmin(member, spaceId)).toBe(true); + + // a non-member is not an admin + const stranger = await v7(); + await core.createUser(stranger, `s_${rand(8)}@example.com`); + expect(await core.isSpaceAdmin(stranger, spaceId)).toBe(false); +}); + test("group grants are inherited transitively (Model 2)", async () => { // a user who is ONLY a group member (no direct principal_space row, no direct // grant) inherits the group's grant via build_tree_access. diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index 9c2df15..46e6fd6 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -8,6 +8,7 @@ import type { Group, GroupMember, GroupMembership, + MemberSpace, Principal, PrincipalKind, Space, @@ -29,6 +30,8 @@ export interface CoreStore { getSpace(slug: string): Promise; /** All spaces (e.g. for the embedding worker to discover me_ schemas). */ listSpaces(): Promise; + /** Spaces a member belongs to (directly or via a group), with admin flag. */ + listSpacesForMember(memberId: string): Promise; createUser(id: string, name: string): Promise; createAgent(ownerId: string, name: string, id?: string): Promise; @@ -208,6 +211,15 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return rows.map(mapSpace); }, + async listSpacesForMember(memberId) { + const rows = await sql` + select * from ${sch}.list_spaces_for_member(${memberId}) + `; + return rows.map( + (r): MemberSpace => ({ ...mapSpace(r), admin: Boolean(r.admin) }), + ); + }, + async createUser(id, name) { const [row] = await sql`select ${sch}.create_user(${id}, ${name}) as id`; if (!row) throw new Error("create_user returned no row"); diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts index 053d925..bf31948 100644 --- a/packages/engine/core/index.ts +++ b/packages/engine/core/index.ts @@ -13,6 +13,7 @@ export type { Group, GroupMember, GroupMembership, + MemberSpace, Principal, PrincipalKind, Space, diff --git a/packages/engine/core/types.ts b/packages/engine/core/types.ts index e04df6c..17c3838 100644 --- a/packages/engine/core/types.ts +++ b/packages/engine/core/types.ts @@ -35,6 +35,11 @@ export interface Space { updatedAt: Date | null; } +/** A space a principal belongs to, with the principal's direct-membership admin flag. */ +export interface MemberSpace extends Space { + admin: boolean; +} + export interface Principal { id: string; kind: PrincipalKind; diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 091a23f..3035c61 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -91,7 +91,7 @@ "!*.test.ts" ], "scripts": { - "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/member.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", + "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/member.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", "build:types": "tsc -p tsconfig.build.json", "build": "../../bun run build:js && ../../bun run build:types", "prepublishOnly": "../../bun run build" diff --git a/packages/protocol/user/index.ts b/packages/protocol/user/index.ts index 4e6f9fa..efb1b48 100644 --- a/packages/protocol/user/index.ts +++ b/packages/protocol/user/index.ts @@ -15,8 +15,10 @@ import { agentRenameParams, agentRenameResult, } from "./agent.ts"; +import { spaceListParams, spaceListResult } from "./space.ts"; export * from "./agent.ts"; +export * from "./space.ts"; function method( params: TParams, @@ -25,12 +27,14 @@ function method( return { params, result }; } -/** User RPC method contract (agent lifecycle). */ +/** User RPC method contract (agent lifecycle + space discovery). */ export const userMethods = { "agent.create": method(agentCreateParams, agentCreateResult), "agent.list": method(agentListParams, agentListResult), "agent.rename": method(agentRenameParams, agentRenameResult), "agent.delete": method(agentDeleteParams, agentDeleteResult), + + "space.list": method(spaceListParams, spaceListResult), } as const; export type UserMethodName = keyof typeof userMethods; diff --git a/packages/protocol/user/space.ts b/packages/protocol/user/space.ts new file mode 100644 index 0000000..9f555ac --- /dev/null +++ b/packages/protocol/user/space.ts @@ -0,0 +1,28 @@ +/** + * Space method schemas (space.*) for the user RPC. + * + * Lets a logged-in user discover the spaces they belong to — used by the CLI to + * select the X-Me-Space the rest of the commands are scoped to. + */ +import { z } from "zod"; + +export const memberSpaceResponse = z.object({ + id: z.string(), + slug: z.string(), + name: z.string(), + language: z.string(), + /** Whether the user is a (direct) admin of the space. */ + admin: z.boolean(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type MemberSpaceResponse = z.infer; + +// space.list — the caller's spaces +export const spaceListParams = z.object({}); +export type SpaceListParams = z.infer; + +export const spaceListResult = z.object({ + spaces: z.array(memberSpaceResponse), +}); +export type SpaceListResult = z.infer; diff --git a/packages/server/rpc/user/agent.integration.test.ts b/packages/server/rpc/user/agent.integration.test.ts index 864b6db..46ec92d 100644 --- a/packages/server/rpc/user/agent.integration.test.ts +++ b/packages/server/rpc/user/agent.integration.test.ts @@ -125,3 +125,56 @@ test("rename/delete of a non-existent agent → NOT_FOUND", async () => { "NOT_FOUND", ); }); + +test("space.list returns the spaces the user belongs to (with admin flag)", async () => { + const core = coreStore(sql, coreSchema); + const spaceId = await core.createSpace(rand(12), "My Space"); + await core.addPrincipalToSpace(spaceId, userId, true); + + const res = await call<{ + spaces: { id: string; name: string; admin: boolean }[]; + }>("space.list", {}); + const mine = res.spaces.find((s) => s.id === spaceId); + expect(mine).toBeDefined(); + expect(mine?.name).toBe("My Space"); + expect(mine?.admin).toBe(true); + + // a brand-new user with no memberships sees no spaces + const other = await makeUser(); + const otherList = await call<{ spaces: unknown[] }>("space.list", {}, other); + expect(otherList.spaces).toHaveLength(0); +}); + +test("space.list includes spaces reached only via group membership", async () => { + const core = coreStore(sql, coreSchema); + const spaceId = await core.createSpace(rand(12), "Group Space"); + const groupId = await core.createGroup(spaceId, "team"); + // the user is NOT added to principal_space — only to a group in the space + await core.addGroupMember(spaceId, groupId, userId); + + const res = await call<{ spaces: { id: string; admin: boolean }[] }>( + "space.list", + {}, + ); + const mine = res.spaces.find((s) => s.id === spaceId); + expect(mine).toBeDefined(); // group membership confers space membership + expect(mine?.admin).toBe(false); // but not direct-membership admin +}); + +test("space.list reflects admin inherited via an admin group", async () => { + const core = coreStore(sql, coreSchema); + const spaceId = await core.createSpace(rand(12), "Admin Group Space"); + const groupId = await core.createGroup(spaceId, "admins"); + // designate the group itself as an admin member of the space + await core.addPrincipalToSpace(spaceId, groupId, true); + // the user is only in that group (no direct principal_space row) + await core.addGroupMember(spaceId, groupId, userId); + + const res = await call<{ spaces: { id: string; admin: boolean }[] }>( + "space.list", + {}, + ); + const mine = res.spaces.find((s) => s.id === spaceId); + expect(mine).toBeDefined(); + expect(mine?.admin).toBe(true); // admin transfers transitively through the group +}); diff --git a/packages/server/rpc/user/index.ts b/packages/server/rpc/user/index.ts index 7783693..804f633 100644 --- a/packages/server/rpc/user/index.ts +++ b/packages/server/rpc/user/index.ts @@ -2,7 +2,9 @@ * User RPC method registry — served at `/api/v1/user/rpc` (session-only, * user-scoped). Currently the lifecycle of a user's agents. */ +import type { MethodRegistry } from "../types"; import { agentMethods } from "./agent"; +import { spaceMethods } from "./space"; export { assertUserRpcContext, @@ -10,4 +12,8 @@ export { type UserRpcContext, } from "./types"; -export const userMethods = agentMethods; +/** The user-endpoint registry: agent lifecycle + space discovery. */ +export const userMethods: MethodRegistry = new Map([ + ...agentMethods, + ...spaceMethods, +]); diff --git a/packages/server/rpc/user/space.ts b/packages/server/rpc/user/space.ts new file mode 100644 index 0000000..70b80cd --- /dev/null +++ b/packages/server/rpc/user/space.ts @@ -0,0 +1,42 @@ +/** + * Space handlers (space.*) for the user RPC. + * + * User-scoped space discovery: the spaces the calling user belongs to. The CLI + * uses this to pick the X-Me-Space that scopes the rest of its commands. + */ +import type { MemberSpace } from "@memory.build/engine/core"; +import type { + MemberSpaceResponse, + SpaceListParams, + SpaceListResult, +} from "@memory.build/protocol/user"; +import { spaceListParams } from "@memory.build/protocol/user"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +function toMemberSpaceResponse(s: MemberSpace): MemberSpaceResponse { + return { + id: s.id, + slug: s.slug, + name: s.name, + language: s.language, + admin: s.admin, + createdAt: s.createdAt.toISOString(), + updatedAt: s.updatedAt?.toISOString() ?? null, + }; +} + +async function spaceList( + _params: SpaceListParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const spaces = await ctx.core.listSpacesForMember(ctx.userId); + return { spaces: spaces.map(toMemberSpaceResponse) }; +} + +export const spaceMethods = buildRegistry() + .register("space.list", spaceListParams, spaceList) + .build(); From 8ec3c0b7a7aefd7842c9be1bc34e95661318a60e Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 13:35:21 +0200 Subject: [PATCH 047/156] feat(core): space.create on the user endpoint (4E-S2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let a logged-in user create a new space from the CLI. Atomic in one transaction: insert the core.space row, provision the me_ data schema (provisionSpace), add the creator as an admin member, and grant them owner@root. The new space starts with an empty tree. - protocol user/space.ts: space.create params/result ({ name } → { id, slug }). - UserRpcContext gains db + coreSchema; the router supplies them so the handler can run the provisioning transaction. - integration test: schema provisioned, space appears in space.list as admin, creator is root owner. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/protocol/user/index.ts | 8 +++- packages/protocol/user/space.ts | 11 +++++ packages/server/router.ts | 2 +- .../server/rpc/user/agent.integration.test.ts | 37 +++++++++++++++- packages/server/rpc/user/space.ts | 43 ++++++++++++++++++- packages/server/rpc/user/types.ts | 5 +++ 6 files changed, 100 insertions(+), 6 deletions(-) diff --git a/packages/protocol/user/index.ts b/packages/protocol/user/index.ts index efb1b48..18981dc 100644 --- a/packages/protocol/user/index.ts +++ b/packages/protocol/user/index.ts @@ -15,7 +15,12 @@ import { agentRenameParams, agentRenameResult, } from "./agent.ts"; -import { spaceListParams, spaceListResult } from "./space.ts"; +import { + spaceCreateParams, + spaceCreateResult, + spaceListParams, + spaceListResult, +} from "./space.ts"; export * from "./agent.ts"; export * from "./space.ts"; @@ -35,6 +40,7 @@ export const userMethods = { "agent.delete": method(agentDeleteParams, agentDeleteResult), "space.list": method(spaceListParams, spaceListResult), + "space.create": method(spaceCreateParams, spaceCreateResult), } as const; export type UserMethodName = keyof typeof userMethods; diff --git a/packages/protocol/user/space.ts b/packages/protocol/user/space.ts index 9f555ac..3402ba9 100644 --- a/packages/protocol/user/space.ts +++ b/packages/protocol/user/space.ts @@ -5,6 +5,7 @@ * select the X-Me-Space the rest of the commands are scoped to. */ import { z } from "zod"; +import { nameSchema } from "../fields.ts"; export const memberSpaceResponse = z.object({ id: z.string(), @@ -26,3 +27,13 @@ export const spaceListResult = z.object({ spaces: z.array(memberSpaceResponse), }); export type SpaceListResult = z.infer; + +// space.create — create a new space; the caller becomes admin + owner@root +export const spaceCreateParams = z.object({ name: nameSchema }); +export type SpaceCreateParams = z.infer; + +export const spaceCreateResult = z.object({ + id: z.string(), + slug: z.string(), +}); +export type SpaceCreateResult = z.infer; diff --git a/packages/server/router.ts b/packages/server/router.ts index 1792bce..ff6c6cf 100644 --- a/packages/server/router.ts +++ b/packages/server/router.ts @@ -236,7 +236,7 @@ export function createRouter(ctx: ServerContext): Router { if (!result.ok) { return result.error; } - return { core, userId: result.context.userId }; + return { core, userId: result.context.userId, db, coreSchema }; }); /** diff --git a/packages/server/rpc/user/agent.integration.test.ts b/packages/server/rpc/user/agent.integration.test.ts index 46ec92d..8f1ac60 100644 --- a/packages/server/rpc/user/agent.integration.test.ts +++ b/packages/server/rpc/user/agent.integration.test.ts @@ -4,8 +4,8 @@ // bun test --timeout 30000 \ // packages/server/rpc/user/agent.integration.test.ts import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; -import { migrateCore } from "@memory.build/database"; -import { coreStore } from "@memory.build/engine/core"; +import { bootstrapSpaceDatabase, migrateCore } from "@memory.build/database"; +import { ACCESS, coreStore, ROOT_PATH } from "@memory.build/engine/core"; import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; import postgres, { type Sql } from "postgres"; import type { HandlerContext } from "../types"; @@ -26,6 +26,7 @@ const rand = (n: number) => { let sql: Sql; let coreSchema: string; let userId: string; +const createdSpaceSchemas: string[] = []; function call( method: string, @@ -38,6 +39,8 @@ function call( request: new Request("http://localhost/api/v1/user/rpc"), core: coreStore(sql, coreSchema), userId: asUser, + db: sql, + coreSchema, } as unknown as HandlerContext; return registered.handler(params, context) as Promise; } @@ -62,10 +65,14 @@ async function makeUser(): Promise { beforeAll(async () => { sql = postgres(URL, { onnotice: () => {} }); coreSchema = `core_test_${rand(8)}`; + await bootstrapSpaceDatabase(sql); // extensions for me_ (space.create) await migrateCore(sql, { schema: coreSchema }); }); afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); await sql.end(); }); @@ -161,6 +168,32 @@ test("space.list includes spaces reached only via group membership", async () => expect(mine?.admin).toBe(false); // but not direct-membership admin }); +test("space.create provisions a space the caller owns + admins", async () => { + const res = await call<{ id: string; slug: string }>("space.create", { + name: "Fresh Space", + }); + createdSpaceSchemas.push(`me_${res.slug}`); + expect(res.slug).toMatch(/^[a-z0-9]{12}$/); + + // the me_ data schema was provisioned + const [row] = await sql.unsafe( + `select exists (select 1 from information_schema.schemata where schema_name = $1) as e`, + [`me_${res.slug}`], + ); + expect(Boolean(row?.e)).toBe(true); + + // it shows up in the caller's spaces, as admin + const list = await call<{ spaces: { id: string; admin: boolean }[] }>( + "space.list", + {}, + ); + expect(list.spaces.find((s) => s.id === res.id)?.admin).toBe(true); + + // and the creator is owner of the root path + const ta = await coreStore(sql, coreSchema).buildTreeAccess(userId, res.id); + expect(ta).toContainEqual({ tree_path: ROOT_PATH, access: ACCESS.owner }); +}); + test("space.list reflects admin inherited via an admin group", async () => { const core = coreStore(sql, coreSchema); const spaceId = await core.createSpace(rand(12), "Admin Group Space"); diff --git a/packages/server/rpc/user/space.ts b/packages/server/rpc/user/space.ts index 70b80cd..0067d40 100644 --- a/packages/server/rpc/user/space.ts +++ b/packages/server/rpc/user/space.ts @@ -4,13 +4,25 @@ * User-scoped space discovery: the spaces the calling user belongs to. The CLI * uses this to pick the X-Me-Space that scopes the rest of its commands. */ -import type { MemberSpace } from "@memory.build/engine/core"; +import { generateSlug, provisionSpace } from "@memory.build/database"; +import { + ACCESS, + coreStore, + type MemberSpace, + ROOT_PATH, +} from "@memory.build/engine/core"; import type { MemberSpaceResponse, + SpaceCreateParams, + SpaceCreateResult, SpaceListParams, SpaceListResult, } from "@memory.build/protocol/user"; -import { spaceListParams } from "@memory.build/protocol/user"; +import { + spaceCreateParams, + spaceListParams, +} from "@memory.build/protocol/user"; +import type { Sql } from "postgres"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; import { assertUserRpcContext, type UserRpcContext } from "./types"; @@ -37,6 +49,33 @@ async function spaceList( return { spaces: spaces.map(toMemberSpaceResponse) }; } +/** + * Create a new space and make the calling user its admin + owner of the root. + * Atomic: the core.space row, the me_ data schema, the membership, and the + * owner grant all land in one transaction (any failure rolls the schema back). + * The new space starts with an empty tree. + */ +async function spaceCreate( + params: SpaceCreateParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const slug = generateSlug(); + + const id = (await ctx.db.begin(async (tx) => { + const core = coreStore(tx as unknown as Sql, ctx.coreSchema); + const spaceId = await core.createSpace(slug, params.name); + await provisionSpace(tx, { slug }); // creates the me_ data schema + await core.addPrincipalToSpace(spaceId, ctx.userId, true); + await core.grantTreeAccess(spaceId, ctx.userId, ROOT_PATH, ACCESS.owner); + return spaceId; + })) as string; + + return { id, slug }; +} + export const spaceMethods = buildRegistry() .register("space.list", spaceListParams, spaceList) + .register("space.create", spaceCreateParams, spaceCreate) .build(); diff --git a/packages/server/rpc/user/types.ts b/packages/server/rpc/user/types.ts index 6268fc7..7687797 100644 --- a/packages/server/rpc/user/types.ts +++ b/packages/server/rpc/user/types.ts @@ -3,6 +3,7 @@ * the calling user manages their own global service accounts (agents). */ import type { CoreStore } from "@memory.build/engine/core"; +import type { Sql } from "postgres"; import type { HandlerContext } from "../types"; export interface UserRpcContext extends HandlerContext { @@ -10,6 +11,10 @@ export interface UserRpcContext extends HandlerContext { core: CoreStore; /** The authenticated user id (== the core user-principal id). */ userId: string; + /** New-model pool — for transactional provisioning (space.create). */ + db: Sql; + /** The core control-plane schema name. */ + coreSchema: string; } export function isUserRpcContext(ctx: HandlerContext): ctx is UserRpcContext { From c24a1a1f28f964bcd445589cc38e605f40dd70f4 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 15:42:40 +0200 Subject: [PATCH 048/156] feat(core): space.rename + space.delete on the user endpoint (4E-S3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round out user-endpoint space management. - core.rename_space / core.delete_space (cascades the space's core rows by slug) + coreStore.renameSpace/deleteSpace. - space.rename: change a space's display name only — the slug (and thus the me_ schema, api keys, and X-Me-Space routing) is immutable, so there's no schema rename. Admin-gated. - space.delete: drop the core.space row (cascade) AND the me_ data schema in one transaction. Admin-gated. - both resolve the space by slug via requireSpaceAdminFor. - integration tests: rename reflected in space.list; delete removes the space + drops the schema; non-admin gets FORBIDDEN. - TODO: consider a distinct space-owner flag to protect destructive ops (delete) vs routine admin management. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 13 ++++ .../core/migrate/idempotent/004_space.sql | 38 ++++++++++ packages/engine/core/db.ts | 18 +++++ packages/protocol/user/index.ts | 6 ++ packages/protocol/user/space.ts | 21 ++++++ .../server/rpc/user/agent.integration.test.ts | 55 +++++++++++++++ packages/server/rpc/user/space.ts | 69 ++++++++++++++++++- 7 files changed, 219 insertions(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index b410514..b676bbf 100644 --- a/TODO.md +++ b/TODO.md @@ -3,6 +3,19 @@ Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). +## Space owner flag (protect destructive ops) + +`space.delete` (and `space.rename`) are currently gated on space-admin +(`principal_space.admin`, which is transitive through admin groups). Deleting a +space drops the whole `me_` schema (all memories), so any admin — including +one who inherited admin via a group — can destroy everything. + +- [ ] Consider a distinct space-**owner** notion (e.g. a `principal_space.owner` + flag, or treating owner@root as the gate) for the truly destructive ops + (delete, and maybe transfer-ownership), keeping admin for routine + structural management (groups, members, grants). Decide whether owner is + transitive through groups (probably not) and how ownership is transferred. + ## Reconsider: api keys for users (not just agents) Keys are currently agent-only (`apiKey.create` is gated by `requireOwnedAgent`; diff --git a/packages/database/core/migrate/idempotent/004_space.sql b/packages/database/core/migrate/idempotent/004_space.sql index c7f6bac..1e7ffa7 100644 --- a/packages/database/core/migrate/idempotent/004_space.sql +++ b/packages/database/core/migrate/idempotent/004_space.sql @@ -15,6 +15,44 @@ $func$ language sql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; +------------------------------------------------------------------------------- +-- rename_space +------------------------------------------------------------------------------- +create or replace function {{schema}}.rename_space +( _slug text +, _name text +) +returns bool +as $func$ + with u as + ( + update {{schema}}.space set name = _name where slug = _slug returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_space +-- Deletes the core.space row; FKs cascade its memberships, groups, grants, and +-- group memberships. The me_ data schema is dropped separately by the +-- caller (DDL). Returns true if a space with this slug existed. +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_space +( _slug text +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.space where slug = _slug returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + ------------------------------------------------------------------------------- -- get_space ------------------------------------------------------------------------------- diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index 46e6fd6..d84e835 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -32,6 +32,13 @@ export interface CoreStore { listSpaces(): Promise; /** Spaces a member belongs to (directly or via a group), with admin flag. */ listSpacesForMember(memberId: string): Promise; + /** Rename a space (by slug). Returns true if it existed. */ + renameSpace(slug: string, name: string): Promise; + /** + * Delete a space's core row (cascades memberships/groups/grants). The + * me_ data schema must be dropped separately. Returns true if it existed. + */ + deleteSpace(slug: string): Promise; createUser(id: string, name: string): Promise; createAgent(ownerId: string, name: string, id?: string): Promise; @@ -220,6 +227,17 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { ); }, + async renameSpace(slug, name) { + const [row] = + await sql`select ${sch}.rename_space(${slug}, ${name}) as ok`; + return Boolean(row?.ok); + }, + + async deleteSpace(slug) { + const [row] = await sql`select ${sch}.delete_space(${slug}) as ok`; + return Boolean(row?.ok); + }, + async createUser(id, name) { const [row] = await sql`select ${sch}.create_user(${id}, ${name}) as id`; if (!row) throw new Error("create_user returned no row"); diff --git a/packages/protocol/user/index.ts b/packages/protocol/user/index.ts index 18981dc..11295ee 100644 --- a/packages/protocol/user/index.ts +++ b/packages/protocol/user/index.ts @@ -18,8 +18,12 @@ import { import { spaceCreateParams, spaceCreateResult, + spaceDeleteParams, + spaceDeleteResult, spaceListParams, spaceListResult, + spaceRenameParams, + spaceRenameResult, } from "./space.ts"; export * from "./agent.ts"; @@ -41,6 +45,8 @@ export const userMethods = { "space.list": method(spaceListParams, spaceListResult), "space.create": method(spaceCreateParams, spaceCreateResult), + "space.rename": method(spaceRenameParams, spaceRenameResult), + "space.delete": method(spaceDeleteParams, spaceDeleteResult), } as const; export type UserMethodName = keyof typeof userMethods; diff --git a/packages/protocol/user/space.ts b/packages/protocol/user/space.ts index 3402ba9..1f48e5b 100644 --- a/packages/protocol/user/space.ts +++ b/packages/protocol/user/space.ts @@ -37,3 +37,24 @@ export const spaceCreateResult = z.object({ slug: z.string(), }); export type SpaceCreateResult = z.infer; + +/** A space's slug (12-char routing key). */ +const spaceSlugSchema = z.string().regex(/^[a-z0-9]{12}$/); + +// space.rename — change a space's display name (admin only). The slug (and +// thus the me_ schema, api keys, and routing) is immutable. +export const spaceRenameParams = z.object({ + slug: spaceSlugSchema, + name: nameSchema, +}); +export type SpaceRenameParams = z.infer; + +export const spaceRenameResult = z.object({ renamed: z.boolean() }); +export type SpaceRenameResult = z.infer; + +// space.delete — delete a space + its data schema (admin only) +export const spaceDeleteParams = z.object({ slug: spaceSlugSchema }); +export type SpaceDeleteParams = z.infer; + +export const spaceDeleteResult = z.object({ deleted: z.boolean() }); +export type SpaceDeleteResult = z.infer; diff --git a/packages/server/rpc/user/agent.integration.test.ts b/packages/server/rpc/user/agent.integration.test.ts index 8f1ac60..93d2636 100644 --- a/packages/server/rpc/user/agent.integration.test.ts +++ b/packages/server/rpc/user/agent.integration.test.ts @@ -194,6 +194,61 @@ test("space.create provisions a space the caller owns + admins", async () => { expect(ta).toContainEqual({ tree_path: ROOT_PATH, access: ACCESS.owner }); }); +test("space.rename renames; space.delete removes the space + schema", async () => { + const created = await call<{ id: string; slug: string }>("space.create", { + name: "Temp Space", + }); + const schema = `me_${created.slug}`; + createdSpaceSchemas.push(schema); + + // rename + expect( + ( + await call<{ renamed: boolean }>("space.rename", { + slug: created.slug, + name: "Renamed Space", + }) + ).renamed, + ).toBe(true); + const after = await call<{ spaces: { id: string; name: string }[] }>( + "space.list", + {}, + ); + expect(after.spaces.find((s) => s.id === created.id)?.name).toBe( + "Renamed Space", + ); + + // delete: core row gone + data schema dropped + expect( + (await call<{ deleted: boolean }>("space.delete", { slug: created.slug })) + .deleted, + ).toBe(true); + const gone = await call<{ spaces: { id: string }[] }>("space.list", {}); + expect(gone.spaces.some((s) => s.id === created.id)).toBe(false); + const [row] = await sql.unsafe( + `select exists (select 1 from information_schema.schemata where schema_name = $1) as e`, + [schema], + ); + expect(Boolean(row?.e)).toBe(false); +}); + +test("space.rename/delete require space admin", async () => { + const created = await call<{ id: string; slug: string }>("space.create", { + name: "Owned Space", + }); + createdSpaceSchemas.push(`me_${created.slug}`); + // a different user who is not a member/admin + const intruder = await makeUser(); + await expectAppError( + call("space.rename", { slug: created.slug, name: "Hijacked" }, intruder), + "FORBIDDEN", + ); + await expectAppError( + call("space.delete", { slug: created.slug }, intruder), + "FORBIDDEN", + ); +}); + test("space.list reflects admin inherited via an admin group", async () => { const core = coreStore(sql, coreSchema); const spaceId = await core.createSpace(rand(12), "Admin Group Space"); diff --git a/packages/server/rpc/user/space.ts b/packages/server/rpc/user/space.ts index 0067d40..f98acaa 100644 --- a/packages/server/rpc/user/space.ts +++ b/packages/server/rpc/user/space.ts @@ -4,29 +4,56 @@ * User-scoped space discovery: the spaces the calling user belongs to. The CLI * uses this to pick the X-Me-Space that scopes the rest of its commands. */ -import { generateSlug, provisionSpace } from "@memory.build/database"; +import { + generateSlug, + provisionSpace, + slugToSchema, +} from "@memory.build/database"; import { ACCESS, coreStore, type MemberSpace, ROOT_PATH, + type Space, } from "@memory.build/engine/core"; import type { MemberSpaceResponse, SpaceCreateParams, SpaceCreateResult, + SpaceDeleteParams, + SpaceDeleteResult, SpaceListParams, SpaceListResult, + SpaceRenameParams, + SpaceRenameResult, } from "@memory.build/protocol/user"; import { spaceCreateParams, + spaceDeleteParams, spaceListParams, + spaceRenameParams, } from "@memory.build/protocol/user"; import type { Sql } from "postgres"; +import { AppError } from "../errors"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; import { assertUserRpcContext, type UserRpcContext } from "./types"; +/** Resolve a space by slug and require the caller to be its admin. */ +async function requireSpaceAdminFor( + ctx: UserRpcContext, + slug: string, +): Promise { + const space = await ctx.core.getSpace(slug); + if (!space) { + throw new AppError("NOT_FOUND", `Space not found: ${slug}`); + } + if (!(await ctx.core.isSpaceAdmin(ctx.userId, space.id))) { + throw new AppError("FORBIDDEN", "This action requires being a space admin"); + } + return space; +} + function toMemberSpaceResponse(s: MemberSpace): MemberSpaceResponse { return { id: s.id, @@ -75,7 +102,47 @@ async function spaceCreate( return { id, slug }; } +/** Rename a space's display name (admin only); the slug is immutable. */ +async function spaceRename( + params: SpaceRenameParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireSpaceAdminFor(ctx, params.slug); + const renamed = await ctx.core.renameSpace(params.slug, params.name); + return { renamed }; +} + +/** + * Delete a space (admin only): drop its core row (cascading memberships/groups/ + * grants) and its me_ data schema, atomically. + */ +async function spaceDelete( + params: SpaceDeleteParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const space = await requireSpaceAdminFor(ctx, params.slug); + + const deleted = (await ctx.db.begin(async (tx) => { + const core = coreStore(tx as unknown as Sql, ctx.coreSchema); + const ok = await core.deleteSpace(space.slug); + // slug came from the DB (validated by the slug check constraint); safe to + // interpolate into the DDL. + await tx.unsafe( + `drop schema if exists ${slugToSchema(space.slug)} cascade`, + ); + return ok; + })) as boolean; + + return { deleted }; +} + export const spaceMethods = buildRegistry() .register("space.list", spaceListParams, spaceList) .register("space.create", spaceCreateParams, spaceCreate) + .register("space.rename", spaceRenameParams, spaceRename) + .register("space.delete", spaceDeleteParams, spaceDelete) .build(); From 5de0924b3ec3f85bd48c511ae5cefcea7f25e2f3 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 15:48:33 +0200 Subject: [PATCH 049/156] feat(server): group.listForMember allows an agent's owner (4E-S4) group.listForMember now permits the caller to list the groups of an agent they own (so `me agent group list ` works), in addition to their own memberships; listing anyone else's still requires space-admin. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/server/rpc/memory/group.ts | 10 ++++-- .../rpc/memory/management.integration.test.ts | 35 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/packages/server/rpc/memory/group.ts b/packages/server/rpc/memory/group.ts index 486241f..c0b59fc 100644 --- a/packages/server/rpc/memory/group.ts +++ b/packages/server/rpc/memory/group.ts @@ -34,6 +34,7 @@ import { AppError } from "../errors"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; import { + callerOwnsAgent, guardCore, requireGroupAdmin, requireSpaceAdmin, @@ -158,9 +159,12 @@ async function groupListForMember( ): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; - // Anyone may see their OWN group memberships; seeing another principal's - // requires space-admin authority. - if (params.memberId !== ctx.principalId) { + // You may see your OWN memberships, or those of an agent you own (so + // `me agent group list` works); seeing anyone else's requires space-admin. + if ( + params.memberId !== ctx.principalId && + !(await callerOwnsAgent(ctx, params.memberId)) + ) { requireSpaceAdmin(ctx); } const groups = await ctx.core.listGroupsForMember( diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index c3799b1..ae1d647 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -383,6 +383,41 @@ test("group member management allows a group admin (not a space admin)", async ( ); }); +test("group.listForMember: an agent's owner can list its groups", async () => { + // owner sets up: a member who owns an agent, the agent is in a group + const member = await makeUser(); + const agentId = await makeAgent(member); + await call("member.add", { principalId: agentId }); + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "bots", + }); + await call("group.addMember", { groupId, memberId: agentId }); + + // the member (agent owner, not a space admin) can list their agent's groups + const as = { + principalId: member, + treeAccess: [] as TreeAccess, + admin: false, + }; + const res = await call<{ groups: { groupId: string }[] }>( + "group.listForMember", + { memberId: agentId }, + as, + ); + expect(res.groups.some((g) => g.groupId === groupId)).toBe(true); + + // a stranger who doesn't own the agent cannot + const stranger = await makeUser(); + await expectAppError( + call( + "group.listForMember", + { memberId: agentId }, + { principalId: stranger, treeAccess: [] as TreeAccess, admin: false }, + ), + "FORBIDDEN", + ); +}); + test("group management requires admin — owner@root is not enough", async () => { // a member who owns the whole data tree (owner@root) but is NOT a space admin const member = await makeUser(); From e8eafb8f377524ec63476b4181105e6c19577234 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:03:10 +0200 Subject: [PATCH 050/156] feat(client): memory + user clients; principal-centric roster (4E-C) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the new-model client surface for the two new RPC endpoints: - createMemoryClient → /api/v1/memory/rpc (session or api-key), selecting the active space via the X-Me-Space header; namespaces memory + principal / group / grant / apiKey. - createUserClient → /api/v1/user/rpc (session-only); namespaces agent + space. TransportConfig gains a `headers` field (merged after the built-ins) so the memory client can carry X-Me-Space. Header constants move out of version.ts into protocol/headers.ts (CLIENT_VERSION_HEADER, SPACE_HEADER). Rename the space roster to principal-centric naming: "principal" is the union (user | agent | group), while "member"/memberId stays reserved for the user/agent sense (group members, api-key holders). - protocol: space/member.ts → space/principal.ts; member.* methods → principal.*; spaceMemberResponse → spacePrincipalResponse; list result field members → principals. - engine core: SpaceMember → SpacePrincipal; listSpaceMembers/mapSpaceMember → listSpacePrincipals/mapSpacePrincipal; SQL list_space_members → list_space_principals. - server: rpc/memory/member.ts → principal.ts; toSpaceMemberResponse → toSpacePrincipalResponse. - client: MemberNamespace → PrincipalNamespace; client.member → client.principal. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/client/index.ts | 19 +- packages/client/memory.test.ts | 72 ++++++ packages/client/memory.ts | 229 ++++++++++++++++++ packages/client/transport.ts | 10 +- packages/client/user.ts | 100 ++++++++ .../migrate/idempotent/006_membership.sql | 4 +- packages/engine/core/core.integration.test.ts | 14 +- packages/engine/core/db.ts | 14 +- packages/engine/core/index.ts | 2 +- packages/engine/core/types.ts | 2 +- packages/protocol/headers.ts | 17 ++ packages/protocol/index.ts | 2 + packages/protocol/package.json | 7 +- packages/protocol/space/group.ts | 4 +- packages/protocol/space/index.ts | 34 +-- packages/protocol/space/member.ts | 87 ------- packages/protocol/space/principal.ts | 91 +++++++ packages/protocol/version.ts | 12 +- .../server/middleware/authenticate-space.ts | 4 +- packages/server/rpc/memory/index.ts | 4 +- .../rpc/memory/management.integration.test.ts | 62 +++-- .../rpc/memory/{member.ts => principal.ts} | 70 +++--- packages/server/rpc/memory/support.ts | 16 +- 23 files changed, 665 insertions(+), 211 deletions(-) create mode 100644 packages/client/memory.test.ts create mode 100644 packages/client/memory.ts create mode 100644 packages/client/user.ts create mode 100644 packages/protocol/headers.ts delete mode 100644 packages/protocol/space/member.ts create mode 100644 packages/protocol/space/principal.ts rename packages/server/rpc/memory/{member.ts => principal.ts} (58%) diff --git a/packages/client/index.ts b/packages/client/index.ts index 7784c91..56be121 100644 --- a/packages/client/index.ts +++ b/packages/client/index.ts @@ -62,11 +62,26 @@ export type { RoleNamespace, UserNamespace, } from "./engine.ts"; -// Engine client (primary) +// Engine client (legacy; removed in Phase 5) export { createClient } from "./engine.ts"; - // Errors export { isRpcError, RpcError } from "./errors.ts"; +// Memory client (new model: space data-plane + management) +export { + createMemoryClient, + type GroupNamespace, + type MemoryClient, + type MemoryClientOptions, + type PrincipalNamespace, +} from "./memory.ts"; +// User client (new model: agent lifecycle + space discovery/management) +export { + type AgentNamespace, + createUserClient, + type SpaceNamespace, + type UserClient, + type UserClientOptions, +} from "./user.ts"; // Version compatibility check export { type CheckServerVersionOptions, diff --git a/packages/client/memory.test.ts b/packages/client/memory.test.ts new file mode 100644 index 0000000..cb4867a --- /dev/null +++ b/packages/client/memory.test.ts @@ -0,0 +1,72 @@ +import { afterEach, expect, test } from "bun:test"; +import { createMemoryClient } from "./memory.ts"; +import { createUserClient } from "./user.ts"; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function captureFetch() { + const captured = { headers: {} as Record, url: "" }; + globalThis.fetch = (async ( + input: string | URL | Request, + init?: RequestInit, + ) => { + captured.url = typeof input === "string" ? input : input.toString(); + const headers = init?.headers as Record | undefined; + if (headers) Object.assign(captured.headers, headers); + return new Response(JSON.stringify({ jsonrpc: "2.0", id: 1, result: {} }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }) as typeof fetch; + return captured; +} + +test("memory client sends X-Me-Space and Bearer token to the memory endpoint", async () => { + const captured = captureFetch(); + const client = createMemoryClient({ + url: "https://api.example.com", + token: "sess-tok", + space: "abc123def456", + retries: 0, + }); + + await client.principal.list({}); + + expect(captured.url).toBe("https://api.example.com/api/v1/memory/rpc"); + expect(captured.headers["X-Me-Space"]).toBe("abc123def456"); + expect(captured.headers.Authorization).toBe("Bearer sess-tok"); +}); + +test("memory client setSpace updates the X-Me-Space header", async () => { + const captured = captureFetch(); + const client = createMemoryClient({ + url: "https://api.example.com", + token: "t", + space: "aaaaaaaaaaaa", + retries: 0, + }); + client.setSpace("bbbbbbbbbbbb"); + + await client.memory.tree(); + + expect(captured.headers["X-Me-Space"]).toBe("bbbbbbbbbbbb"); +}); + +test("user client targets the user endpoint with no X-Me-Space", async () => { + const captured = captureFetch(); + const client = createUserClient({ + url: "https://api.example.com", + token: "sess-tok", + retries: 0, + }); + + await client.space.list(); + + expect(captured.url).toBe("https://api.example.com/api/v1/user/rpc"); + expect(captured.headers["X-Me-Space"]).toBeUndefined(); + expect(captured.headers.Authorization).toBe("Bearer sess-tok"); +}); diff --git a/packages/client/memory.ts b/packages/client/memory.ts new file mode 100644 index 0000000..4faf185 --- /dev/null +++ b/packages/client/memory.ts @@ -0,0 +1,229 @@ +/** + * Memory client — the space data-plane + management client. + * + * Talks to POST /api/v1/memory/rpc, authenticated by a session token (human) or + * an api key (agent), with the active space selected via the X-Me-Space header. + * Namespaces: memory (data plane) + principal / group / grant / apiKey (management). + * + * @example + * ```ts + * const me = createMemoryClient({ token: sessionToken, space: "abc123def456" }); + * await me.memory.create({ content: "hello", tree: "notes" }); + * await me.principal.list({}); + * ``` + */ +import type { + MemoryBatchCreateParams, + MemoryBatchCreateResult, + MemoryCountTreeParams, + MemoryCountTreeResult, + MemoryCreateParams, + MemoryDeleteParams, + MemoryDeleteResult, + MemoryDeleteTreeParams, + MemoryDeleteTreeResult, + MemoryGetParams, + MemoryMoveParams, + MemoryMoveResult, + MemoryResponse, + MemorySearchParams, + MemorySearchResult, + MemoryTreeParams, + MemoryTreeResult, + MemoryUpdateParams, +} from "@memory.build/protocol/engine/memory"; +import { SPACE_HEADER } from "@memory.build/protocol/headers"; +import type { + ApiKeyCreateParams, + ApiKeyCreateResult, + ApiKeyDeleteParams, + ApiKeyDeleteResult, + ApiKeyGetParams, + ApiKeyGetResult, + ApiKeyListParams, + ApiKeyListResult, + GrantListParams, + GrantListResult, + GrantRemoveParams, + GrantRemoveResult, + GrantSetParams, + GrantSetResult, + GroupAddMemberParams, + GroupAddMemberResult, + GroupCreateParams, + GroupCreateResult, + GroupDeleteParams, + GroupDeleteResult, + GroupListForMemberParams, + GroupListForMemberResult, + GroupListMembersParams, + GroupListMembersResult, + GroupListParams, + GroupListResult, + GroupRemoveMemberParams, + GroupRemoveMemberResult, + GroupRenameParams, + GroupRenameResult, + PrincipalAddParams, + PrincipalAddResult, + PrincipalListParams, + PrincipalListResult, + PrincipalRemoveParams, + PrincipalRemoveResult, + PrincipalResolveByEmailParams, + PrincipalResolveByEmailResult, +} from "@memory.build/protocol/space"; +import { rpcCall, type TransportConfig } from "./transport.ts"; + +export interface MemoryClientOptions { + /** Base URL of the server (default: "https://api.memory.build") */ + url?: string; + /** Memory RPC endpoint path (default: "/api/v1/memory/rpc") */ + rpcPath?: string; + /** Bearer token: a session token (human) or an api key (agent). */ + token?: string; + /** The active space slug, sent as X-Me-Space. */ + space?: string; + /** Request timeout in milliseconds (default: 30000) */ + timeout?: number; + /** Maximum retry attempts for transient failures (default: 3) */ + retries?: number; + /** CLIENT_VERSION of the caller (sent as X-Client-Version). */ + clientVersion?: string; +} + +export interface MemoryNamespace { + create(params: MemoryCreateParams): Promise; + batchCreate( + params: MemoryBatchCreateParams, + ): Promise; + get(params: MemoryGetParams): Promise; + update(params: MemoryUpdateParams): Promise; + delete(params: MemoryDeleteParams): Promise; + search(params: MemorySearchParams): Promise; + tree(params?: MemoryTreeParams): Promise; + move(params: MemoryMoveParams): Promise; + deleteTree(params: MemoryDeleteTreeParams): Promise; + countTree(params: MemoryCountTreeParams): Promise; +} + +export interface PrincipalNamespace { + list(params?: PrincipalListParams): Promise; + add(params: PrincipalAddParams): Promise; + remove(params: PrincipalRemoveParams): Promise; + resolveByEmail( + params: PrincipalResolveByEmailParams, + ): Promise; +} + +export interface GroupNamespace { + create(params: GroupCreateParams): Promise; + list(params?: GroupListParams): Promise; + rename(params: GroupRenameParams): Promise; + delete(params: GroupDeleteParams): Promise; + addMember(params: GroupAddMemberParams): Promise; + removeMember( + params: GroupRemoveMemberParams, + ): Promise; + listMembers(params: GroupListMembersParams): Promise; + listForMember( + params: GroupListForMemberParams, + ): Promise; +} + +export interface GrantNamespace { + set(params: GrantSetParams): Promise; + remove(params: GrantRemoveParams): Promise; + list(params?: GrantListParams): Promise; +} + +export interface ApiKeyNamespace { + create(params: ApiKeyCreateParams): Promise; + list(params: ApiKeyListParams): Promise; + get(params: ApiKeyGetParams): Promise; + delete(params: ApiKeyDeleteParams): Promise; +} + +export interface MemoryClient { + memory: MemoryNamespace; + principal: PrincipalNamespace; + group: GroupNamespace; + grant: GrantNamespace; + apiKey: ApiKeyNamespace; + + /** Update the bearer token (session or api key) at runtime. */ + setToken(token: string): void; + /** Update the active space slug (X-Me-Space) at runtime. */ + setSpace(space: string): void; +} + +const DEFAULT_URL = "https://api.memory.build"; +const MEMORY_RPC_PATH = "/api/v1/memory/rpc"; +const DEFAULT_TIMEOUT = 30_000; +const DEFAULT_RETRIES = 3; + +export function createMemoryClient( + options: MemoryClientOptions = {}, +): MemoryClient { + const config: TransportConfig = { + url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), + path: options.rpcPath ?? MEMORY_RPC_PATH, + token: options.token, + timeout: options.timeout ?? DEFAULT_TIMEOUT, + retries: options.retries ?? DEFAULT_RETRIES, + clientVersion: options.clientVersion, + headers: options.space ? { [SPACE_HEADER]: options.space } : undefined, + }; + + function rpc(method: string, params: unknown): Promise { + return rpcCall(config, method, params); + } + + return { + memory: { + create: (p) => rpc("memory.create", p), + batchCreate: (p) => rpc("memory.batchCreate", p), + get: (p) => rpc("memory.get", p), + update: (p) => rpc("memory.update", p), + delete: (p) => rpc("memory.delete", p), + search: (p) => rpc("memory.search", p), + tree: (p) => rpc("memory.tree", p ?? {}), + move: (p) => rpc("memory.move", p), + deleteTree: (p) => rpc("memory.deleteTree", p), + countTree: (p) => rpc("memory.countTree", p), + }, + principal: { + list: (p) => rpc("principal.list", p ?? {}), + add: (p) => rpc("principal.add", p), + remove: (p) => rpc("principal.remove", p), + resolveByEmail: (p) => rpc("principal.resolveByEmail", p), + }, + group: { + create: (p) => rpc("group.create", p), + list: (p) => rpc("group.list", p ?? {}), + rename: (p) => rpc("group.rename", p), + delete: (p) => rpc("group.delete", p), + addMember: (p) => rpc("group.addMember", p), + removeMember: (p) => rpc("group.removeMember", p), + listMembers: (p) => rpc("group.listMembers", p), + listForMember: (p) => rpc("group.listForMember", p), + }, + grant: { + set: (p) => rpc("grant.set", p), + remove: (p) => rpc("grant.remove", p), + list: (p) => rpc("grant.list", p ?? {}), + }, + apiKey: { + create: (p) => rpc("apiKey.create", p), + list: (p) => rpc("apiKey.list", p), + get: (p) => rpc("apiKey.get", p), + delete: (p) => rpc("apiKey.delete", p), + }, + setToken(token: string) { + config.token = token; + }, + setSpace(space: string) { + config.headers = { ...config.headers, [SPACE_HEADER]: space }; + }, + }; +} diff --git a/packages/client/transport.ts b/packages/client/transport.ts index c70bc8d..020b2ce 100644 --- a/packages/client/transport.ts +++ b/packages/client/transport.ts @@ -4,7 +4,7 @@ * Handles HTTP communication, retry logic with exponential backoff, * timeouts, and JSON-RPC envelope formatting. */ -import { CLIENT_VERSION_HEADER } from "@memory.build/protocol"; +import { CLIENT_VERSION_HEADER } from "@memory.build/protocol/headers"; import type { JsonRpcErrorResponse, JsonRpcResponse, @@ -35,6 +35,11 @@ export interface TransportConfig { * before dispatch. Optional — older callers without this set still work. */ clientVersion?: string; + /** + * Extra headers sent on every RPC (e.g. `X-Me-Space` to select the space for + * the memory endpoint). Merged after the built-in headers. + */ + headers?: Record; } // ============================================================================= @@ -94,6 +99,9 @@ export async function rpcCall( if (config.clientVersion) { headers[CLIENT_VERSION_HEADER] = config.clientVersion; } + if (config.headers) { + Object.assign(headers, config.headers); + } let lastError: Error | undefined; diff --git a/packages/client/user.ts b/packages/client/user.ts new file mode 100644 index 0000000..5dab48b --- /dev/null +++ b/packages/client/user.ts @@ -0,0 +1,100 @@ +/** + * User client — session-only, user-scoped operations. + * + * Talks to POST /api/v1/user/rpc, authenticated by a session token. Namespaces: + * agent (a user's global service accounts) and space (discover/create/manage the + * user's spaces — used by the CLI to pick the active X-Me-Space). + */ +import type { + AgentCreateParams, + AgentCreateResult, + AgentDeleteParams, + AgentDeleteResult, + AgentListParams, + AgentListResult, + AgentRenameParams, + AgentRenameResult, + SpaceCreateParams, + SpaceCreateResult, + SpaceDeleteParams, + SpaceDeleteResult, + SpaceListParams, + SpaceListResult, + SpaceRenameParams, + SpaceRenameResult, +} from "@memory.build/protocol/user"; +import { rpcCall, type TransportConfig } from "./transport.ts"; + +export interface UserClientOptions { + /** Base URL of the server (default: "https://api.memory.build") */ + url?: string; + /** User RPC endpoint path (default: "/api/v1/user/rpc") */ + rpcPath?: string; + /** Session token (humans only). */ + token?: string; + /** Request timeout in milliseconds (default: 30000) */ + timeout?: number; + /** Maximum retry attempts for transient failures (default: 3) */ + retries?: number; + /** CLIENT_VERSION of the caller (sent as X-Client-Version). */ + clientVersion?: string; +} + +export interface AgentNamespace { + create(params: AgentCreateParams): Promise; + list(params?: AgentListParams): Promise; + rename(params: AgentRenameParams): Promise; + delete(params: AgentDeleteParams): Promise; +} + +export interface SpaceNamespace { + list(params?: SpaceListParams): Promise; + create(params: SpaceCreateParams): Promise; + rename(params: SpaceRenameParams): Promise; + delete(params: SpaceDeleteParams): Promise; +} + +export interface UserClient { + agent: AgentNamespace; + space: SpaceNamespace; + /** Update the session token at runtime. */ + setToken(token: string): void; +} + +const DEFAULT_URL = "https://api.memory.build"; +const USER_RPC_PATH = "/api/v1/user/rpc"; +const DEFAULT_TIMEOUT = 30_000; +const DEFAULT_RETRIES = 3; + +export function createUserClient(options: UserClientOptions = {}): UserClient { + const config: TransportConfig = { + url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), + path: options.rpcPath ?? USER_RPC_PATH, + token: options.token, + timeout: options.timeout ?? DEFAULT_TIMEOUT, + retries: options.retries ?? DEFAULT_RETRIES, + clientVersion: options.clientVersion, + }; + + function rpc(method: string, params: unknown): Promise { + return rpcCall(config, method, params); + } + + return { + agent: { + create: (p) => rpc("agent.create", p), + list: (p) => rpc("agent.list", p ?? {}), + rename: (p) => rpc("agent.rename", p), + delete: (p) => rpc("agent.delete", p), + }, + space: { + list: (p) => rpc("space.list", p ?? {}), + create: (p) => rpc("space.create", p), + rename: (p) => rpc("space.rename", p), + delete: (p) => rpc("space.delete", p), + }, + setToken(token: string) { + config.token = token; + }, + }; +} diff --git a/packages/database/core/migrate/idempotent/006_membership.sql b/packages/database/core/migrate/idempotent/006_membership.sql index eb2f2eb..26da08b 100644 --- a/packages/database/core/migrate/idempotent/006_membership.sql +++ b/packages/database/core/migrate/idempotent/006_membership.sql @@ -100,7 +100,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- list_space_members +-- list_space_principals -- Principals that belong to a space, deduplicated: either added directly -- (principal_space) or reached through a group in the space (group_member) — -- group membership confers space access, so both count. `direct` is true when @@ -108,7 +108,7 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- admin flag (false for group-only members). Optional kind filter -- ('u' | 'a' | 'g'); null returns all. ------------------------------------------------------------------------------- -create or replace function {{schema}}.list_space_members +create or replace function {{schema}}.list_space_principals ( _space_id uuid , _kind text default null ) diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts index f3d9ed2..2454a69 100644 --- a/packages/engine/core/core.integration.test.ts +++ b/packages/engine/core/core.integration.test.ts @@ -68,25 +68,25 @@ test("renamePrincipal refuses to rename users", async () => { expect((await core.getPrincipal(userId))?.name).toBe(userName); }); -test("listSpaceMembers lists direct members with admin flag and kind filter", async () => { - const all = await core.listSpaceMembers(spaceId); +test("listSpacePrincipals lists direct principals with admin flag and kind filter", async () => { + const all = await core.listSpacePrincipals(spaceId); expect(all).toHaveLength(1); expect(all[0]?.id).toBe(userId); expect(all[0]?.direct).toBe(true); expect(all[0]?.admin).toBe(true); - expect(await core.listSpaceMembers(spaceId, "u")).toHaveLength(1); - expect(await core.listSpaceMembers(spaceId, "g")).toHaveLength(0); + expect(await core.listSpacePrincipals(spaceId, "u")).toHaveLength(1); + expect(await core.listSpacePrincipals(spaceId, "g")).toHaveLength(0); }); -test("listSpaceMembers includes group-only members (flagged direct=false)", async () => { +test("listSpacePrincipals includes group-only principals (flagged direct=false)", async () => { // a second user who is NOT added to the space directly, only via a group const groupOnlyId = await v7(); await core.createUser(groupOnlyId, `grouponly_${rand(8)}@example.com`); const groupId = await core.createGroup(spaceId, "team"); await core.addGroupMember(spaceId, groupId, groupOnlyId); - const members = await core.listSpaceMembers(spaceId, "u"); + const members = await core.listSpacePrincipals(spaceId, "u"); const byId = Object.fromEntries(members.map((m) => [m.id, m])); // owner is a direct member expect(byId[userId]?.direct).toBe(true); @@ -99,7 +99,7 @@ test("listSpaceMembers includes group-only members (flagged direct=false)", asyn test("agents appear as space members of kind 'a'", async () => { const agentId = await core.createAgent(userId, `agent-${rand(6)}`); await core.addPrincipalToSpace(spaceId, agentId); - const agents = await core.listSpaceMembers(spaceId, "a"); + const agents = await core.listSpacePrincipals(spaceId, "a"); expect(agents).toHaveLength(1); expect(agents[0]?.id).toBe(agentId); expect(agents[0]?.ownerId).toBe(userId); diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index d84e835..28a3050 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -12,7 +12,7 @@ import type { Principal, PrincipalKind, Space, - SpaceMember, + SpacePrincipal, TreeAccess, TreeGrant, ValidatedApiKey, @@ -51,10 +51,10 @@ export interface CoreStore { deletePrincipal(id: string): Promise; /** Principals in a space — directly or via a group (each flagged `direct`). */ - listSpaceMembers( + listSpacePrincipals( spaceId: string, kind?: PrincipalKind, - ): Promise; + ): Promise; /** Whether a principal is an admin of a space (agents are never admins). */ isSpaceAdmin(principalId: string, spaceId: string): Promise; /** Whether a member is an admin of a group (agents are never group admins). */ @@ -163,7 +163,7 @@ function mapPrincipal(row: Record): Principal { }; } -function mapSpaceMember(row: Record): SpaceMember { +function mapSpacePrincipal(row: Record): SpacePrincipal { return { id: row.id as string, kind: row.kind as PrincipalKind, @@ -282,11 +282,11 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return Boolean(row?.ok); }, - async listSpaceMembers(spaceId, kind) { + async listSpacePrincipals(spaceId, kind) { const rows = await sql` - select * from ${sch}.list_space_members(${spaceId}, ${kind ?? null}) + select * from ${sch}.list_space_principals(${spaceId}, ${kind ?? null}) `; - return rows.map(mapSpaceMember); + return rows.map(mapSpacePrincipal); }, async listSpaceGroups(spaceId) { diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts index bf31948..2cc8f0c 100644 --- a/packages/engine/core/index.ts +++ b/packages/engine/core/index.ts @@ -17,7 +17,7 @@ export type { Principal, PrincipalKind, Space, - SpaceMember, + SpacePrincipal, TreeAccess, TreeGrant, ValidatedApiKey, diff --git a/packages/engine/core/types.ts b/packages/engine/core/types.ts index 17c3838..2dfc604 100644 --- a/packages/engine/core/types.ts +++ b/packages/engine/core/types.ts @@ -79,7 +79,7 @@ export interface ValidatedApiKey { * `direct` is true for a direct (principal_space) membership; `admin` is the * direct-membership admin flag (false for group-only members). */ -export interface SpaceMember { +export interface SpacePrincipal { id: string; kind: PrincipalKind; name: string; diff --git a/packages/protocol/headers.ts b/packages/protocol/headers.ts new file mode 100644 index 0000000..fa1377c --- /dev/null +++ b/packages/protocol/headers.ts @@ -0,0 +1,17 @@ +/** + * HTTP header names shared between the server and clients. + */ + +/** + * Header the client uses to advertise its CLIENT_VERSION on every RPC. + * + * The server short-circuits requests with an incompatible client version + * before dispatching them to a handler. + */ +export const CLIENT_VERSION_HEADER = "X-Client-Version"; + +/** + * Header the client sends to select which space a memory-endpoint request + * targets (both session and api-key auth). The value is the active space slug. + */ +export const SPACE_HEADER = "X-Me-Space"; diff --git a/packages/protocol/index.ts b/packages/protocol/index.ts index 6081843..2eadc5c 100644 --- a/packages/protocol/index.ts +++ b/packages/protocol/index.ts @@ -20,6 +20,8 @@ export * from "./engine/index.ts"; export * from "./errors.ts"; // Shared field validators export * from "./fields.ts"; +// HTTP header names +export * from "./headers.ts"; // JSON-RPC 2.0 envelope types export * from "./jsonrpc.ts"; // Version compatibility schemas diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 3035c61..b64dc90 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -29,6 +29,11 @@ "types": "./errors.ts", "import": "./dist/errors.js" }, + "./headers": { + "bun": "./headers.ts", + "types": "./headers.ts", + "import": "./dist/headers.js" + }, "./version": { "bun": "./version.ts", "types": "./version.ts", @@ -91,7 +96,7 @@ "!*.test.ts" ], "scripts": { - "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/member.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", + "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts headers.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/principal.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", "build:types": "tsc -p tsconfig.build.json", "build": "../../bun run build:js && ../../bun run build:types", "prepublishOnly": "../../bun run build" diff --git a/packages/protocol/space/group.ts b/packages/protocol/space/group.ts index 4b686e6..87018f6 100644 --- a/packages/protocol/space/group.ts +++ b/packages/protocol/space/group.ts @@ -3,11 +3,11 @@ * * Groups are space-scoped principals used to bundle members for tree-access * grants. Group membership confers space access (a group member is a space - * member, flagged direct=false in member.list). + * member, flagged direct=false in principal.list). */ import { z } from "zod"; import { nameSchema, uuidv7Schema } from "../fields.ts"; -import { principalKindSchema } from "./member.ts"; +import { principalKindSchema } from "./principal.ts"; export const groupResponse = z.object({ id: z.string(), diff --git a/packages/protocol/space/index.ts b/packages/protocol/space/index.ts index a2c54e3..abb9c3a 100644 --- a/packages/protocol/space/index.ts +++ b/packages/protocol/space/index.ts @@ -45,20 +45,20 @@ import { groupRenameResult, } from "./group.ts"; import { - memberAddParams, - memberAddResult, - memberListParams, - memberListResult, - memberRemoveParams, - memberRemoveResult, - memberResolveByEmailParams, - memberResolveByEmailResult, -} from "./member.ts"; + principalAddParams, + principalAddResult, + principalListParams, + principalListResult, + principalRemoveParams, + principalRemoveResult, + principalResolveByEmailParams, + principalResolveByEmailResult, +} from "./principal.ts"; export * from "./api-key.ts"; export * from "./grant.ts"; export * from "./group.ts"; -export * from "./member.ts"; +export * from "./principal.ts"; function method( params: TParams, @@ -72,13 +72,13 @@ function method( * Served on the memory endpoint together with the memory.* methods. */ export const spaceMethods = { - // Membership (4) - "member.list": method(memberListParams, memberListResult), - "member.add": method(memberAddParams, memberAddResult), - "member.remove": method(memberRemoveParams, memberRemoveResult), - "member.resolveByEmail": method( - memberResolveByEmailParams, - memberResolveByEmailResult, + // Membership (4) — the space roster holds principals (user | agent | group) + "principal.list": method(principalListParams, principalListResult), + "principal.add": method(principalAddParams, principalAddResult), + "principal.remove": method(principalRemoveParams, principalRemoveResult), + "principal.resolveByEmail": method( + principalResolveByEmailParams, + principalResolveByEmailResult, ), // Groups (8) diff --git a/packages/protocol/space/member.ts b/packages/protocol/space/member.ts deleted file mode 100644 index a2e17b9..0000000 --- a/packages/protocol/space/member.ts +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Space membership method schemas (member.*). - * - * The space management API, served on POST /api/v1/memory/rpc, follows the core - * model: principals (users/agents/groups), space membership, group membership, - * 3-level tree-access grants, and agent api keys. All methods are scoped to the - * space selected by the X-Me-Space header and require space-owner authority. - */ -import { z } from "zod"; -import { emailSchema, nameSchema, uuidv7Schema } from "../fields.ts"; - -/** Principal kind: user / group / agent. */ -export const principalKindSchema = z.enum(["u", "g", "a"]); -export type PrincipalKind = z.infer; - -/** - * A principal that belongs to a space — directly or via a group. - * `direct` is true for a direct membership; `admin` is the direct-membership - * admin flag (false for group-only members). - */ -export const spaceMemberResponse = z.object({ - id: z.string(), - kind: principalKindSchema, - name: z.string(), - ownerId: z.string().nullable(), - direct: z.boolean(), - admin: z.boolean(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); -export type SpaceMemberResponse = z.infer; - -/** A resolved principal (used by member.resolveByEmail). */ -export const principalResponse = z.object({ - id: z.string(), - kind: principalKindSchema, - name: z.string(), - ownerId: z.string().nullable(), - spaceId: z.string().nullable(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); -export type PrincipalResponse = z.infer; - -// member.list -export const memberListParams = z.object({ - kind: principalKindSchema.optional().nullable(), -}); -export type MemberListParams = z.infer; - -export const memberListResult = z.object({ - members: z.array(spaceMemberResponse), -}); -export type MemberListResult = z.infer; - -// member.add -export const memberAddParams = z.object({ - principalId: uuidv7Schema, - admin: z.boolean().optional(), -}); -export type MemberAddParams = z.infer; - -export const memberAddResult = z.object({ added: z.boolean() }); -export type MemberAddResult = z.infer; - -// member.remove -export const memberRemoveParams = z.object({ principalId: uuidv7Schema }); -export type MemberRemoveParams = z.infer; - -export const memberRemoveResult = z.object({ removed: z.boolean() }); -export type MemberRemoveResult = z.infer; - -// member.resolveByEmail — find a global user by email (to add them to the space) -export const memberResolveByEmailParams = z.object({ email: emailSchema }); -export type MemberResolveByEmailParams = z.infer< - typeof memberResolveByEmailParams ->; - -export const memberResolveByEmailResult = z.object({ - principal: principalResponse.nullable(), -}); -export type MemberResolveByEmailResult = z.infer< - typeof memberResolveByEmailResult ->; - -// shared by agent.* / group.* mutation results -export { nameSchema }; diff --git a/packages/protocol/space/principal.ts b/packages/protocol/space/principal.ts new file mode 100644 index 0000000..d989c65 --- /dev/null +++ b/packages/protocol/space/principal.ts @@ -0,0 +1,91 @@ +/** + * Space membership method schemas (principal.*). + * + * The space management API, served on POST /api/v1/memory/rpc, follows the core + * model: principals (users/agents/groups), space membership, group membership, + * 3-level tree-access grants, and agent api keys. All methods are scoped to the + * space selected by the X-Me-Space header and require space-owner authority. + * + * "Principal" is the union (user | agent | group); the space roster holds + * principals. "Member" is reserved for the user/agent sense (group members, + * api-key holders). + */ +import { z } from "zod"; +import { emailSchema, nameSchema, uuidv7Schema } from "../fields.ts"; + +/** Principal kind: user / group / agent. */ +export const principalKindSchema = z.enum(["u", "g", "a"]); +export type PrincipalKind = z.infer; + +/** + * A principal that belongs to a space — directly or via a group. + * `direct` is true for a direct membership; `admin` is the direct-membership + * admin flag (false for group-only members). + */ +export const spacePrincipalResponse = z.object({ + id: z.string(), + kind: principalKindSchema, + name: z.string(), + ownerId: z.string().nullable(), + direct: z.boolean(), + admin: z.boolean(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type SpacePrincipalResponse = z.infer; + +/** A resolved principal (used by principal.resolveByEmail). */ +export const principalResponse = z.object({ + id: z.string(), + kind: principalKindSchema, + name: z.string(), + ownerId: z.string().nullable(), + spaceId: z.string().nullable(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type PrincipalResponse = z.infer; + +// principal.list +export const principalListParams = z.object({ + kind: principalKindSchema.optional().nullable(), +}); +export type PrincipalListParams = z.infer; + +export const principalListResult = z.object({ + principals: z.array(spacePrincipalResponse), +}); +export type PrincipalListResult = z.infer; + +// principal.add +export const principalAddParams = z.object({ + principalId: uuidv7Schema, + admin: z.boolean().optional(), +}); +export type PrincipalAddParams = z.infer; + +export const principalAddResult = z.object({ added: z.boolean() }); +export type PrincipalAddResult = z.infer; + +// principal.remove +export const principalRemoveParams = z.object({ principalId: uuidv7Schema }); +export type PrincipalRemoveParams = z.infer; + +export const principalRemoveResult = z.object({ removed: z.boolean() }); +export type PrincipalRemoveResult = z.infer; + +// principal.resolveByEmail — find a global user by email (to add to the space) +export const principalResolveByEmailParams = z.object({ email: emailSchema }); +export type PrincipalResolveByEmailParams = z.infer< + typeof principalResolveByEmailParams +>; + +export const principalResolveByEmailResult = z.object({ + principal: principalResponse.nullable(), +}); +export type PrincipalResolveByEmailResult = z.infer< + typeof principalResolveByEmailResult +>; + +// shared by agent.* / group.* mutation results +export { nameSchema }; diff --git a/packages/protocol/version.ts b/packages/protocol/version.ts index 85b426e..2ddf659 100644 --- a/packages/protocol/version.ts +++ b/packages/protocol/version.ts @@ -13,17 +13,7 @@ */ import { z } from "zod"; -// ============================================================================= -// Headers -// ============================================================================= - -/** - * Header name the client uses to advertise its CLIENT_VERSION on every RPC. - * - * The server short-circuits requests with an incompatible client version - * before dispatching them to a handler. - */ -export const CLIENT_VERSION_HEADER = "X-Client-Version"; +// Header constants (CLIENT_VERSION_HEADER, SPACE_HEADER) live in ./headers.ts. // ============================================================================= // GET /api/v1/version diff --git a/packages/server/middleware/authenticate-space.ts b/packages/server/middleware/authenticate-space.ts index a47d1c5..e68b55a 100644 --- a/packages/server/middleware/authenticate-space.ts +++ b/packages/server/middleware/authenticate-space.ts @@ -22,13 +22,13 @@ import { type TreeAccess, } from "@memory.build/engine/core"; import { type SpaceStore, spaceStore } from "@memory.build/engine/space"; +import { SPACE_HEADER } from "@memory.build/protocol/headers"; import { debug, span } from "@pydantic/logfire-node"; import type { Sql } from "postgres"; import { error, forbidden, unauthorized } from "../util/response"; import { extractBearerToken } from "./authenticate"; -/** Header that selects which space a session / api-key request targets. */ -export const SPACE_HEADER = "X-Me-Space"; +export { SPACE_HEADER }; /** * The authenticated principal + resolved space for a memory RPC request. diff --git a/packages/server/rpc/memory/index.ts b/packages/server/rpc/memory/index.ts index 181b8e9..83987fe 100644 --- a/packages/server/rpc/memory/index.ts +++ b/packages/server/rpc/memory/index.ts @@ -9,8 +9,8 @@ import type { MethodRegistry } from "../types"; import { apiKeyMethods } from "./api-key"; import { grantMethods } from "./grant"; import { groupMethods } from "./group"; -import { memberMethods } from "./member"; import { memoryDataMethods } from "./memory"; +import { principalMethods } from "./principal"; export { assertSpaceRpcContext, @@ -24,7 +24,7 @@ export { */ export const memoryMethods: MethodRegistry = new Map([ ...memoryDataMethods, - ...memberMethods, + ...principalMethods, ...groupMethods, ...grantMethods, ...apiKeyMethods, diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index ae1d647..eb103a8 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -84,7 +84,7 @@ async function makeUser(): Promise { /** * Create a global agent owned by `owner` (the user-endpoint operation), returning - * its id. Not yet a member of any space — member.add brings it in. + * its id. Not yet a member of any space — principal.add brings it in. */ function makeAgent(owner: string): Promise { return engineCore @@ -130,32 +130,35 @@ beforeEach(async () => { .buildTreeAccess(r.userId, r.spaceId); }); -test("member: list / resolveByEmail / add / remove", async () => { - const listed = await call<{ members: { id: string; admin: boolean }[] }>( - "member.list", +test("principal: list / resolveByEmail / add / remove", async () => { + const listed = await call<{ principals: { id: string; admin: boolean }[] }>( + "principal.list", {}, ); - expect(listed.members.some((m) => m.id === ownerId && m.admin)).toBe(true); + expect(listed.principals.some((m) => m.id === ownerId && m.admin)).toBe(true); const resolved = await call<{ principal: { id: string } | null }>( - "member.resolveByEmail", + "principal.resolveByEmail", { email: ownerEmail }, ); expect(resolved.principal?.id).toBe(ownerId); const other = await makeUser(); expect( - (await call<{ added: boolean }>("member.add", { principalId: other })) + (await call<{ added: boolean }>("principal.add", { principalId: other })) .added, ).toBe(true); expect( - (await call<{ members: { id: string }[] }>("member.list", {})).members.some( - (m) => m.id === other, - ), + ( + await call<{ principals: { id: string }[] }>("principal.list", {}) + ).principals.some((m) => m.id === other), ).toBe(true); expect( - (await call<{ removed: boolean }>("member.remove", { principalId: other })) - .removed, + ( + await call<{ removed: boolean }>("principal.remove", { + principalId: other, + }) + ).removed, ).toBe(true); }); @@ -227,7 +230,7 @@ test("apiKey: create (agent-only) / list / get / delete", async () => { // agent lifecycle is the user endpoint's job; here the owner brings an agent // into the space, then mints its (space-bound) key. const agentId = await makeAgent(ownerId); - await call("member.add", { principalId: agentId }); + await call("principal.add", { principalId: agentId }); const created = await call<{ id: string; key: string }>("apiKey.create", { agentId, name: "ci", @@ -269,14 +272,14 @@ test("roster/group management requires admin or owner", async () => { treeAccess: [{ tree_path: "sub", access: 2 }] as TreeAccess, admin: false, }; - await expectAppError(call("member.list", {}, as), "FORBIDDEN"); + await expectAppError(call("principal.list", {}, as), "FORBIDDEN"); await expectAppError(call("group.create", { name: "x" }, as), "FORBIDDEN"); }); test("a space admin (without owner@root) has management authority", async () => { // an admin member with only read access, no owner grant anywhere const adminMember = await makeUser(); - await call("member.add", { principalId: adminMember, admin: true }); + await call("principal.add", { principalId: adminMember, admin: true }); await call("grant.set", { principalId: adminMember, treePath: "x", @@ -289,7 +292,8 @@ test("a space admin (without owner@root) has management authority", async () => // can manage the roster and grant anywhere despite holding no owner grant expect( - (await call<{ members: unknown[] }>("member.list", {}, as)).members.length, + (await call<{ principals: unknown[] }>("principal.list", {}, as)).principals + .length, ).toBeGreaterThan(0); const stranger = await makeUser(); expect( @@ -387,7 +391,7 @@ test("group.listForMember: an agent's owner can list its groups", async () => { // owner sets up: a member who owns an agent, the agent is in a group const member = await makeUser(); const agentId = await makeAgent(member); - await call("member.add", { principalId: agentId }); + await call("principal.add", { principalId: agentId }); const { id: groupId } = await call<{ id: string }>("group.create", { name: "bots", }); @@ -421,7 +425,7 @@ test("group.listForMember: an agent's owner can list its groups", async () => { test("group management requires admin — owner@root is not enough", async () => { // a member who owns the whole data tree (owner@root) but is NOT a space admin const member = await makeUser(); - await call("member.add", { principalId: member }); + await call("principal.add", { principalId: member }); await call("grant.set", { principalId: member, treePath: "", access: 3 }); const ta = await engineCore .coreStore(sql, coreSchema) @@ -430,7 +434,8 @@ test("group management requires admin — owner@root is not enough", async () => // owner@root can manage the roster and grant access (it's their data) expect( - (await call<{ members: unknown[] }>("member.list", {}, as)).members.length, + (await call<{ principals: unknown[] }>("principal.list", {}, as)).principals + .length, ).toBeGreaterThan(0); // but groups are structural — admin only await expectAppError(call("group.create", { name: "g" }, as), "FORBIDDEN"); @@ -439,7 +444,7 @@ test("group management requires admin — owner@root is not enough", async () => test("grant authority is path-scoped: a subtree owner delegates within it", async () => { // a member who owns "proj" (not the root) can manage access under proj only const member = await makeUser(); - await call("member.add", { principalId: member }); + await call("principal.add", { principalId: member }); await call("grant.set", { principalId: member, treePath: "proj", access: 3 }); const memberTA = await engineCore .coreStore(sql, coreSchema) @@ -478,7 +483,7 @@ test("grant authority is path-scoped: a subtree owner delegates within it", asyn test("self-service: a non-owner member brings their own agent into the space", async () => { // owner onboards a second user with write (not owner) on a subtree const member = await makeUser(); - await call("member.add", { principalId: member }); + await call("principal.add", { principalId: member }); await call("grant.set", { principalId: member, treePath: "proj", access: 2 }); const memberTA = await engineCore .coreStore(sql, coreSchema) @@ -486,11 +491,16 @@ test("self-service: a non-owner member brings their own agent into the space", a const as = { principalId: member, treeAccess: memberTA }; // the member created their agent on the user endpoint (simulated via core); - // they bring it into the space (self-service member.add) without owner rights + // they bring it into the space (self-service principal.add) without owner rights const agentId = await makeAgent(member); expect( - (await call<{ added: boolean }>("member.add", { principalId: agentId }, as)) - .added, + ( + await call<{ added: boolean }>( + "principal.add", + { principalId: agentId }, + as, + ) + ).added, ).toBe(true); // and mint its key + self-grant it (capped by their own access) @@ -511,10 +521,10 @@ test("self-service: a non-owner member brings their own agent into the space", a ).toBe(true); // but the member cannot manage the roster, add a stranger, or grant to others - await expectAppError(call("member.list", {}, as), "FORBIDDEN"); + await expectAppError(call("principal.list", {}, as), "FORBIDDEN"); const stranger = await makeUser(); await expectAppError( - call("member.add", { principalId: stranger }, as), + call("principal.add", { principalId: stranger }, as), "FORBIDDEN", ); await expectAppError( diff --git a/packages/server/rpc/memory/member.ts b/packages/server/rpc/memory/principal.ts similarity index 58% rename from packages/server/rpc/memory/member.ts rename to packages/server/rpc/memory/principal.ts index 76974ea..286c925 100644 --- a/packages/server/rpc/memory/member.ts +++ b/packages/server/rpc/memory/principal.ts @@ -1,21 +1,21 @@ /** - * Space membership handlers (member.*) — the space roster (principal_space). + * Space membership handlers (principal.*) — the space roster (principal_space). */ import type { - MemberAddParams, - MemberAddResult, - MemberListParams, - MemberListResult, - MemberRemoveParams, - MemberRemoveResult, - MemberResolveByEmailParams, - MemberResolveByEmailResult, + PrincipalAddParams, + PrincipalAddResult, + PrincipalListParams, + PrincipalListResult, + PrincipalRemoveParams, + PrincipalRemoveResult, + PrincipalResolveByEmailParams, + PrincipalResolveByEmailResult, } from "@memory.build/protocol/space"; import { - memberAddParams, - memberListParams, - memberRemoveParams, - memberResolveByEmailParams, + principalAddParams, + principalListParams, + principalRemoveParams, + principalResolveByEmailParams, } from "@memory.build/protocol/space"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; @@ -24,28 +24,28 @@ import { guardCore, requireSpaceManager, toPrincipalResponse, - toSpaceMemberResponse, + toSpacePrincipalResponse, } from "./support"; import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; -async function memberList( - params: MemberListParams, +async function principalList( + params: PrincipalListParams, context: HandlerContext, -): Promise { +): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; requireSpaceManager(ctx); - const members = await ctx.core.listSpaceMembers( + const principals = await ctx.core.listSpacePrincipals( ctx.space.id, params.kind ?? undefined, ); - return { members: members.map(toSpaceMemberResponse) }; + return { principals: principals.map(toSpacePrincipalResponse) }; } -async function memberAdd( - params: MemberAddParams, +async function principalAdd( + params: PrincipalAddParams, context: HandlerContext, -): Promise { +): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; // Bringing your OWN agent into a space is self-service (it stays capped by @@ -67,10 +67,10 @@ async function memberAdd( return { added: true }; } -async function memberRemove( - params: MemberRemoveParams, +async function principalRemove( + params: PrincipalRemoveParams, context: HandlerContext, -): Promise { +): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; requireSpaceManager(ctx); @@ -80,10 +80,10 @@ async function memberRemove( return { removed }; } -async function memberResolveByEmail( - params: MemberResolveByEmailParams, +async function principalResolveByEmail( + params: PrincipalResolveByEmailParams, context: HandlerContext, -): Promise { +): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; requireSpaceManager(ctx); @@ -91,13 +91,13 @@ async function memberResolveByEmail( return { principal: principal ? toPrincipalResponse(principal) : null }; } -export const memberMethods = buildRegistry() - .register("member.list", memberListParams, memberList) - .register("member.add", memberAddParams, memberAdd) - .register("member.remove", memberRemoveParams, memberRemove) +export const principalMethods = buildRegistry() + .register("principal.list", principalListParams, principalList) + .register("principal.add", principalAddParams, principalAdd) + .register("principal.remove", principalRemoveParams, principalRemove) .register( - "member.resolveByEmail", - memberResolveByEmailParams, - memberResolveByEmail, + "principal.resolveByEmail", + principalResolveByEmailParams, + principalResolveByEmail, ) .build(); diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts index ca17e09..5f98a40 100644 --- a/packages/server/rpc/memory/support.ts +++ b/packages/server/rpc/memory/support.ts @@ -10,7 +10,7 @@ import type { GroupMember, GroupMembership, Principal, - SpaceMember, + SpacePrincipal, TreeGrant, } from "@memory.build/engine/core"; import { ACCESS, ROOT_PATH } from "@memory.build/engine/core"; @@ -20,7 +20,7 @@ import type { GroupMembershipResponse, GroupResponse, PrincipalResponse, - SpaceMemberResponse, + SpacePrincipalResponse, TreeGrantResponse, } from "@memory.build/protocol/space"; import { guardCore } from "../core-error"; @@ -135,7 +135,7 @@ export async function callerOwnsAgent( context: SpaceRpcContext, principalId: string, ): Promise { - const agents = await context.core.listSpaceMembers(context.space.id, "a"); + const agents = await context.core.listSpacePrincipals(context.space.id, "a"); const agent = agents.find((a) => a.id === principalId); return agent !== undefined && agent.ownerId === context.principalId; } @@ -148,7 +148,7 @@ export async function requireOwnedAgent( context: SpaceRpcContext, agentId: string, ): Promise { - const agents = await context.core.listSpaceMembers(context.space.id, "a"); + const agents = await context.core.listSpacePrincipals(context.space.id, "a"); const agent = agents.find((a) => a.id === agentId); if (!agent) { throw new AppError( @@ -163,8 +163,8 @@ export async function requireOwnedAgent( /** * True if `principalId` is an agent owned by the caller, checked globally (not - * scoped to the current space). Used by member.add so a member can bring their - * OWN agent into a space before it is a member there. + * scoped to the current space). Used by principal.add so a member can bring + * their OWN agent into a space before it is a member there. */ export async function callerOwnsAgentGlobal( context: SpaceRpcContext, @@ -182,7 +182,9 @@ export async function callerOwnsAgentGlobal( // Serializers (Date → ISO) // ============================================================================= -export function toSpaceMemberResponse(m: SpaceMember): SpaceMemberResponse { +export function toSpacePrincipalResponse( + m: SpacePrincipal, +): SpacePrincipalResponse { return { id: m.id, kind: m.kind, From 541d629886c4955331356ee9bf8b1b231834f8de Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:14:50 +0200 Subject: [PATCH 051/156] feat(cli): space-centric credential + client foundation (4E-CLI-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive groundwork for the CLI cutover to the new model; legacy engine commands are untouched and still work. - credentials.ts: store the active space (active_space) per server alongside the session token; setActiveSpace / resolveSpace (--space > ME_SPACE > stored). Api keys are never persisted — an agent key only ever comes from ME_API_KEY, since keys are for agents that run elsewhere (apiKey.create prints the key once). TODO noted to move the session token into the OS keychain with a 0600-file fallback. - client.ts: createMemoryClient / createUserClient factories (inject CLIENT_VERSION) + type re-exports. - util.ts: requireSpace guard for the space-scoped commands. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/client.ts | 29 +++++++++++++++ packages/cli/credentials.ts | 73 ++++++++++++++++++++++++++++++++----- packages/cli/util.ts | 20 ++++++++++ 3 files changed, 112 insertions(+), 10 deletions(-) diff --git a/packages/cli/client.ts b/packages/cli/client.ts index 268297a..68d5401 100644 --- a/packages/cli/client.ts +++ b/packages/cli/client.ts @@ -14,8 +14,14 @@ import { createAccountsClient as baseCreateAccountsClient, createAuthClient as baseCreateAuthClient, createClient as baseCreateClient, + createMemoryClient as baseCreateMemoryClient, + createUserClient as baseCreateUserClient, type ClientOptions, type EngineClient, + type MemoryClient, + type MemoryClientOptions, + type UserClient, + type UserClientOptions, } from "@memory.build/client"; import { CLIENT_VERSION } from "../../version"; @@ -49,6 +55,25 @@ export function createAuthClient(options: AuthClientOptions = {}): AuthClient { return baseCreateAuthClient(options); } +/** + * Memory client factory (new model: space data-plane + management) with + * `clientVersion: CLIENT_VERSION` injected. Talks to /api/v1/memory/rpc with the + * active space carried as X-Me-Space. + */ +export function createMemoryClient( + options: MemoryClientOptions = {}, +): MemoryClient { + return baseCreateMemoryClient({ clientVersion: CLIENT_VERSION, ...options }); +} + +/** + * User client factory (new model: agent lifecycle + space discovery) with + * `clientVersion: CLIENT_VERSION` injected. Talks to /api/v1/user/rpc. + */ +export function createUserClient(options: UserClientOptions = {}): UserClient { + return baseCreateUserClient({ clientVersion: CLIENT_VERSION, ...options }); +} + // Re-export types and helpers used across the CLI. Pass-through so command // files don't need to dual-import from "@memory.build/client". export { @@ -62,5 +87,9 @@ export { DeviceFlowError, type EngineClient, isRpcError, + type MemoryClient, + type MemoryClientOptions, RpcError, + type UserClient, + type UserClientOptions, } from "@memory.build/client"; diff --git a/packages/cli/credentials.ts b/packages/cli/credentials.ts index a45916a..2166fdd 100644 --- a/packages/cli/credentials.ts +++ b/packages/cli/credentials.ts @@ -1,22 +1,30 @@ /** - * Credential storage — multi-server, multi-engine credential management. + * Credential storage — multi-server, multi-space credential management. * - * Stores session tokens and per-engine API keys in + * Stores the session token (humans) and per-space agent API keys in * $XDG_CONFIG_HOME/me/credentials.yaml (default: ~/.config/me/). * + * The file holds the human session token and the active space; it never stores + * api keys. Api keys are for agents, which run elsewhere and receive their key + * via the `ME_API_KEY` env var (or pasted into their MCP config). `apiKey.create` + * prints the key once — the operator places it where the agent runs. + * * File format: * ```yaml * default_server: https://api.memory.build * servers: * https://api.memory.build: - * session_token: "..." - * active_engine: "abc123defg45" - * engines: - * abc123defg45: - * api_key: "me.abc123defg45.xxxx.yyyy" - * xyz789qwer12: - * api_key: "me.xyz789qwer12.xxxx.yyyy" + * session_token: "..." # human session (used with X-Me-Space) + * active_space: "abc123def456" # active space slug (the X-Me-Space) * ``` + * + * TODO(keychain): move the session token into the OS keychain (macOS `security`, + * Linux `secret-tool`, Windows credential manager) with a fall back to this 0600 + * file when no keychain is available (CI, headless Linux). The file would then + * hold only non-secret pointers (default_server, active_space). + * + * The `active_engine` / `engines` fields are the legacy engine-model shape; they + * are read for backward compatibility but new logins write the space shape. */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; @@ -34,7 +42,7 @@ export const DEFAULT_SERVER = "https://api.memory.build"; // ============================================================================= /** - * Per-engine credential entry. + * Per-engine credential entry (legacy engine model). */ export interface EngineCredentials { api_key: string; @@ -45,6 +53,9 @@ export interface EngineCredentials { */ export interface ServerCredentials { session_token?: string; + /** Active space slug (sent as X-Me-Space). */ + active_space?: string; + /** Legacy engine model — read for back-compat; new logins write `active_space`. */ active_engine?: string; engines?: Record; } @@ -64,6 +75,9 @@ export interface ResolvedCredentials { server: string; sessionToken?: string; apiKey?: string; + /** Active space slug (the X-Me-Space) — ME_SPACE env > stored active_space. */ + activeSpace?: string; + /** Legacy engine model. */ activeEngine?: string; } @@ -248,6 +262,39 @@ export function getEngineApiKey( return stored.engines?.[engineSlug]?.api_key; } +// ============================================================================= +// Space Operations (new model) +// ============================================================================= + +/** + * Set the active space (the X-Me-Space) for a server. + */ +export function setActiveSpace(server: string, spaceSlug: string): void { + const creds = readCredentials(); + const origin = normalizeOrigin(server); + + if (!creds.servers[origin]) { + creds.servers[origin] = {}; + } + creds.servers[origin].active_space = spaceSlug; + + writeCredentials(creds); +} + +/** + * Resolve the active space slug for a server. + * + * Priority: --space flag > ME_SPACE env > stored active_space. + */ +export function resolveSpace( + server: string, + flagValue?: string, +): string | undefined { + if (flagValue) return flagValue; + if (process.env.ME_SPACE) return process.env.ME_SPACE; + return getServerCredentials(server).active_space; +} + /** * Clear just the session token for a server, leaving any stored engines and * API keys in place. Used after the server tells us the session is expired so @@ -319,10 +366,16 @@ export function resolveCredentials(serverFlag?: string): ResolvedCredentials { ? stored.engines?.[activeEngine]?.api_key : undefined; + // New model: the active space (X-Me-Space); ME_SPACE overrides the stored + // active_space. Api keys are never stored — an agent key only ever comes from + // ME_API_KEY (the legacy engine key remains as a fallback until removed). + const activeSpace = process.env.ME_SPACE ?? stored.active_space; + return { server, sessionToken: process.env.ME_SESSION_TOKEN ?? stored.session_token, apiKey: process.env.ME_API_KEY ?? storedApiKey, + activeSpace, activeEngine, }; } diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 6c63eee..71418f5 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -51,6 +51,26 @@ export function requireEngine( } } +/** + * Ensure an active space (the X-Me-Space) is selected. Exits with an error if + * not. Used by the space-scoped commands (memory, group, access, …). + */ +export function requireSpace( + creds: ResolvedCredentials, + fmt: OutputFormat, +): asserts creds is ResolvedCredentials & { activeSpace: string } { + if (!creds.activeSpace) { + if (fmt === "text") { + clack.log.error( + "No active space. Run 'me space use ' to select one, or set ME_SPACE.", + ); + } else { + output({ error: "No active space" }, fmt, () => {}); + } + process.exit(1); + } +} + interface OrgInfo { id: string; name: string; From 2e079b5bf588e9416e01fd023b6943fa31d6153d Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:22:06 +0200 Subject: [PATCH 052/156] feat(cli): whoami method + login space-selection (4E-CLI-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a session-identity method to the user RPC and cut the CLI's login/whoami over to the new model. - protocol/server/client: new `whoami` method on /api/v1/user/rpc returning the session's identity (id, email, name). UserRpcContext gains the auth store (wired in router.ts); handler resolves via auth.getUser, UNAUTHORIZED when the user row is gone. Client gains userClient.whoami(). - me login [space]: session token + whoami + space.list, then select the active space — explicit slug/name arg, else auto-select a sole space, else prompt (text) / report (json), else suggest `me space create`. Persists active_space; never aborts once the session is stored. - me whoami: identity + server + active space via the user client. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/login.ts | 207 +++++++++--------- packages/cli/commands/whoami.ts | 51 ++--- packages/client/user.ts | 5 + packages/protocol/package.json | 2 +- packages/protocol/user/index.ts | 6 +- packages/protocol/user/whoami.ts | 19 ++ packages/server/router.ts | 2 +- .../server/rpc/user/agent.integration.test.ts | 30 ++- packages/server/rpc/user/agent.ts | 4 +- packages/server/rpc/user/index.ts | 4 +- packages/server/rpc/user/types.ts | 3 + packages/server/rpc/user/whoami.ts | 27 +++ 12 files changed, 218 insertions(+), 142 deletions(-) create mode 100644 packages/protocol/user/whoami.ts create mode 100644 packages/server/rpc/user/whoami.ts diff --git a/packages/cli/commands/login.ts b/packages/cli/commands/login.ts index ff62418..3b78853 100644 --- a/packages/cli/commands/login.ts +++ b/packages/cli/commands/login.ts @@ -1,31 +1,34 @@ /** - * me login — authenticate via OAuth device flow. + * me login [space] — authenticate via OAuth device flow, then pick the active space. * - * 1. User picks a provider (Google/GitHub) - * 2. CLI starts device flow, gets user code + verification URL - * 3. Opens browser (or tells user to visit URL manually) - * 4. Polls until user completes auth - * 5. Stores session token in credentials file - * 6. Fetches and displays identity + * 1. Compatibility check (fail fast before the browser round-trip) + * 2. Start device flow, show user code + URL, open browser + * 3. Poll until the user authorizes → session token + * 4. Store the session token for the server + * 5. Fetch identity (whoami) and the caller's spaces + * 6. Select the active space (the X-Me-Space the rest of the CLI is scoped to): + * - a [space] argument (slug or name) is honored if it matches + * - otherwise auto-select when the user has exactly one space + * - multiple → prompt (text) / report (json); zero → suggest `me space create` */ import * as clack from "@clack/prompts"; import type { OAuthProvider } from "@memory.build/protocol/auth/device-flow"; +import type { MemberSpaceResponse } from "@memory.build/protocol/user"; import { Command } from "commander"; import { CLIENT_VERSION, MIN_SERVER_VERSION } from "../../../version"; import { checkServerVersion, - createAccountsClient, createAuthClient, + createUserClient, DeviceFlowError, RpcError, } from "../client.ts"; import { - getEngineApiKey, resolveServer, - storeApiKey, + setActiveSpace, storeSessionToken, } from "../credentials.ts"; -import { getOutputFormat, output } from "../output.ts"; +import { getOutputFormat, type OutputFormat, output } from "../output.ts"; /** * Attempt to open a URL in the user's default browser. @@ -48,26 +51,37 @@ async function openBrowser(url: string): Promise { } } +/** + * Match a [space] argument against the caller's spaces, by slug (exact) or name + * (case-insensitive). Returns the match, or null when nothing/ambiguous matches. + */ +function matchSpace( + spaces: MemberSpaceResponse[], + input: string, +): MemberSpaceResponse | null { + const bySlug = spaces.find((s) => s.slug === input); + if (bySlug) return bySlug; + const lower = input.toLowerCase(); + const byName = spaces.filter((s) => s.name.toLowerCase() === lower); + return byName.length === 1 ? (byName[0] ?? null) : null; +} + export function createLoginCommand(): Command { return new Command("login") - .description("authenticate with Memory Engine via OAuth") - .action(async (_opts, cmd) => { + .description("authenticate with Memory Engine and select the active space") + .argument("[space]", "space to activate after login (slug or name)") + .action(async (spaceArg: string | undefined, _opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const server = resolveServer(globalOpts.server); const fmt = getOutputFormat(globalOpts); const auth = createAuthClient({ url: server }); - // --- Provider selection --- if (fmt === "text") { clack.intro("me login"); } - // --- Compatibility check --- - // Verify that this CLI and the server agree on a compatible version - // before sending the user through the OAuth round-trip. Failing here - // is much friendlier than failing after they've authorized in their - // browser. + // --- Compatibility check (before the OAuth round-trip) --- try { await checkServerVersion({ url: server, @@ -75,12 +89,7 @@ export function createLoginCommand(): Command { minServerVersion: MIN_SERVER_VERSION, }); } catch (error) { - const msg = - error instanceof RpcError - ? error.message - : error instanceof Error - ? error.message - : String(error); + const msg = error instanceof Error ? error.message : String(error); if (fmt === "text") { clack.log.error(msg); clack.outro("Login failed."); @@ -99,7 +108,7 @@ export function createLoginCommand(): Command { let flow: Awaited>; try { - flow = await auth.startDeviceFlow(provider as OAuthProvider); + flow = await auth.startDeviceFlow(provider); } catch (error) { spin?.stop("Failed to start device flow."); const msg = error instanceof Error ? error.message : String(error); @@ -114,17 +123,15 @@ export function createLoginCommand(): Command { spin?.stop("Device flow started."); - // --- Display code and open browser --- if (fmt === "text") { clack.note( `Code: ${flow.userCode}\nURL: ${flow.verificationUri}`, "Enter this code in your browser", ); } - await openBrowser(flow.verificationUri); - // --- Poll for token --- + // --- Poll for the session token --- spin?.start("Waiting for authorization..."); try { @@ -132,102 +139,37 @@ export function createLoginCommand(): Command { interval: flow.interval, expiresIn: flow.expiresIn, }); - spin?.stop("Authorized!"); - // Store session token storeSessionToken(server, result.sessionToken); - // Fetch identity via accounts client - const accounts = createAccountsClient({ + const user = createUserClient({ url: server, - sessionToken: result.sessionToken, + token: result.sessionToken, }); - const identity = await accounts.me.get(); - - // Auto-select engine if exactly one exists - let engineInfo: { - name: string; - slug: string; - orgName: string; - } | null = null; - let engineCount = 0; - - try { - const { orgs } = await accounts.org.list(); - const allEngines: Array<{ - id: string; - slug: string; - name: string; - orgName: string; - }> = []; - for (const org of orgs) { - const { engines } = await accounts.engine.list({ - orgId: org.id, - }); - for (const e of engines) { - if (e.status === "active") { - allEngines.push({ - id: e.id, - slug: e.slug, - name: e.name, - orgName: org.name, - }); - } - } - } - engineCount = allEngines.length; - - if (allEngines.length === 1 && allEngines[0]) { - const engine = allEngines[0]; - // Check if we already have a key for this engine - const existingKey = getEngineApiKey(server, engine.slug); - if (existingKey) { - // Already have a key — just ensure it's active - const { setActiveEngine } = await import("../credentials.ts"); - setActiveEngine(server, engine.slug); - engineInfo = { - name: engine.name, - slug: engine.slug, - orgName: engine.orgName, - }; - } else { - // Bootstrap access - const setupResult = await accounts.engine.setupAccess({ - engineId: engine.id, - }); - storeApiKey(server, setupResult.engineSlug, setupResult.rawKey); - engineInfo = { - name: setupResult.engineName, - slug: setupResult.engineSlug, - orgName: setupResult.orgName, - }; - } - } - } catch { - // Engine auto-select is best-effort — don't fail login - } + const identity = await user.whoami(); + const { spaces } = await user.space.list(); - output({ server, identity, engine: engineInfo }, fmt, () => { + const active = await selectSpace(server, spaces, spaceArg, fmt); + + output({ server, identity, space: active }, fmt, () => { clack.log.success( `Logged in as ${identity.name} (${identity.email})`, ); clack.log.info(`Server: ${server}`); - if (engineInfo) { - clack.log.info( - `Engine: ${engineInfo.name} (${engineInfo.orgName})`, - ); - } else if (engineCount > 1) { - clack.log.info( - "Multiple engines found. Run 'me engine use' to select one.", - ); + if (active) { + clack.log.info(`Space: ${active.name} (${active.slug})`); + } else if (spaces.length === 0) { + clack.log.info("No spaces yet. Run 'me space create '."); + } else { + clack.log.info("Run 'me space use ' to select a space."); } clack.outro("Done!"); }); } catch (error) { spin?.stop("Authorization failed."); const msg = - error instanceof DeviceFlowError + error instanceof DeviceFlowError || error instanceof RpcError ? error.message : error instanceof Error ? error.message @@ -242,3 +184,52 @@ export function createLoginCommand(): Command { } }); } + +/** + * Resolve and persist the active space after login. Returns the selected space, + * or null when none could be selected (and leaves the active space unchanged). + */ +async function selectSpace( + server: string, + spaces: MemberSpaceResponse[], + spaceArg: string | undefined, + fmt: OutputFormat, +): Promise { + // Explicit argument wins. + if (spaceArg) { + const match = matchSpace(spaces, spaceArg); + if (!match) { + const msg = `No space matching '${spaceArg}'.`; + if (fmt === "text") { + clack.log.error(msg); + for (const s of spaces) console.log(` ${s.name} (${s.slug})`); + } + // Don't abort the whole login — the session is already stored. + return null; + } + setActiveSpace(server, match.slug); + return match; + } + + // Exactly one space → auto-select. + if (spaces.length === 1 && spaces[0]) { + setActiveSpace(server, spaces[0].slug); + return spaces[0]; + } + + // Multiple spaces in an interactive session → prompt. + if (spaces.length > 1 && fmt === "text") { + const choice = await clack.select({ + message: "Select the active space", + options: spaces.map((s) => ({ + value: s.slug, + label: `${s.name} (${s.slug})`, + })), + }); + if (clack.isCancel(choice)) return null; + setActiveSpace(server, choice as string); + return spaces.find((s) => s.slug === choice) ?? null; + } + + return null; +} diff --git a/packages/cli/commands/whoami.ts b/packages/cli/commands/whoami.ts index a00b54d..8230ca9 100644 --- a/packages/cli/commands/whoami.ts +++ b/packages/cli/commands/whoami.ts @@ -1,51 +1,48 @@ /** - * me whoami — show current identity and active engine. + * me whoami — show the current identity, server, and active space. */ import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; +import { createUserClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output } from "../output.ts"; import { handleError, requireSession } from "../util.ts"; export function createWhoamiCommand(): Command { return new Command("whoami") - .description("show current identity and active engine") + .description("show current identity, server, and active space") .action(async (_opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - const accounts = createAccountsClient({ + const user = createUserClient({ url: creds.server, - sessionToken: creds.sessionToken, + token: creds.sessionToken, }); try { - const identity = await accounts.me.get(); + const identity = await user.whoami(); - const data: Record = { - server: creds.server, - identity: { - id: identity.id, - name: identity.name, - email: identity.email, + output( + { + server: creds.server, + identity, + activeSpace: creds.activeSpace ?? null, }, - activeEngine: creds.activeEngine ?? null, - hasApiKey: !!creds.apiKey, - }; - - output(data, fmt, () => { - console.log(` Name: ${identity.name}`); - console.log(` Email: ${identity.email}`); - console.log(` ID: ${identity.id}`); - console.log(` Server: ${creds.server}`); - if (creds.activeEngine) { - console.log(` Engine: ${creds.activeEngine}`); - } else { - console.log(" Engine: (none — run 'me engine use' to select)"); - } - }); + fmt, + () => { + console.log(` Name: ${identity.name}`); + console.log(` Email: ${identity.email}`); + console.log(` ID: ${identity.id}`); + console.log(` Server: ${creds.server}`); + if (creds.activeSpace) { + console.log(` Space: ${creds.activeSpace}`); + } else { + console.log(" Space: (none — run 'me space use ')"); + } + }, + ); } catch (error) { handleError(error, fmt, { sessionServer: creds.server }); } diff --git a/packages/client/user.ts b/packages/client/user.ts index 5dab48b..4757afd 100644 --- a/packages/client/user.ts +++ b/packages/client/user.ts @@ -22,6 +22,8 @@ import type { SpaceListResult, SpaceRenameParams, SpaceRenameResult, + WhoamiParams, + WhoamiResult, } from "@memory.build/protocol/user"; import { rpcCall, type TransportConfig } from "./transport.ts"; @@ -55,6 +57,8 @@ export interface SpaceNamespace { } export interface UserClient { + /** The identity behind the session token. */ + whoami(params?: WhoamiParams): Promise; agent: AgentNamespace; space: SpaceNamespace; /** Update the session token at runtime. */ @@ -81,6 +85,7 @@ export function createUserClient(options: UserClientOptions = {}): UserClient { } return { + whoami: (p) => rpc("whoami", p ?? {}), agent: { create: (p) => rpc("agent.create", p), list: (p) => rpc("agent.list", p ?? {}), diff --git a/packages/protocol/package.json b/packages/protocol/package.json index b64dc90..c3044e7 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -96,7 +96,7 @@ "!*.test.ts" ], "scripts": { - "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts headers.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/principal.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", + "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts headers.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/principal.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts user/whoami.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", "build:types": "tsc -p tsconfig.build.json", "build": "../../bun run build:js && ../../bun run build:types", "prepublishOnly": "../../bun run build" diff --git a/packages/protocol/user/index.ts b/packages/protocol/user/index.ts index 11295ee..cf15570 100644 --- a/packages/protocol/user/index.ts +++ b/packages/protocol/user/index.ts @@ -25,9 +25,11 @@ import { spaceRenameParams, spaceRenameResult, } from "./space.ts"; +import { whoamiParams, whoamiResult } from "./whoami.ts"; export * from "./agent.ts"; export * from "./space.ts"; +export * from "./whoami.ts"; function method( params: TParams, @@ -36,8 +38,10 @@ function method( return { params, result }; } -/** User RPC method contract (agent lifecycle + space discovery). */ +/** User RPC method contract (identity + agent lifecycle + space discovery). */ export const userMethods = { + whoami: method(whoamiParams, whoamiResult), + "agent.create": method(agentCreateParams, agentCreateResult), "agent.list": method(agentListParams, agentListResult), "agent.rename": method(agentRenameParams, agentRenameResult), diff --git a/packages/protocol/user/whoami.ts b/packages/protocol/user/whoami.ts new file mode 100644 index 0000000..385bd20 --- /dev/null +++ b/packages/protocol/user/whoami.ts @@ -0,0 +1,19 @@ +/** + * whoami method schema for the user RPC. + * + * Returns the identity behind the session token — used by the CLI for `me login` + * confirmation and `me whoami`. Session-only (an api key never authenticates the + * user endpoint). + */ +import { z } from "zod"; + +// whoami — the authenticated user's identity +export const whoamiParams = z.object({}); +export type WhoamiParams = z.infer; + +export const whoamiResult = z.object({ + id: z.string(), + email: z.string(), + name: z.string(), +}); +export type WhoamiResult = z.infer; diff --git a/packages/server/router.ts b/packages/server/router.ts index ff6c6cf..2b88f27 100644 --- a/packages/server/router.ts +++ b/packages/server/router.ts @@ -236,7 +236,7 @@ export function createRouter(ctx: ServerContext): Router { if (!result.ok) { return result.error; } - return { core, userId: result.context.userId, db, coreSchema }; + return { core, auth, userId: result.context.userId, db, coreSchema }; }); /** diff --git a/packages/server/rpc/user/agent.integration.test.ts b/packages/server/rpc/user/agent.integration.test.ts index 93d2636..d0486f1 100644 --- a/packages/server/rpc/user/agent.integration.test.ts +++ b/packages/server/rpc/user/agent.integration.test.ts @@ -4,7 +4,12 @@ // bun test --timeout 30000 \ // packages/server/rpc/user/agent.integration.test.ts import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; -import { bootstrapSpaceDatabase, migrateCore } from "@memory.build/database"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; import { ACCESS, coreStore, ROOT_PATH } from "@memory.build/engine/core"; import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; import postgres, { type Sql } from "postgres"; @@ -25,6 +30,7 @@ const rand = (n: number) => { let sql: Sql; let coreSchema: string; +let authSchema: string; let userId: string; const createdSpaceSchemas: string[] = []; @@ -38,6 +44,7 @@ function call( const context = { request: new Request("http://localhost/api/v1/user/rpc"), core: coreStore(sql, coreSchema), + auth: authStore(sql, authSchema), userId: asUser, db: sql, coreSchema, @@ -65,8 +72,10 @@ async function makeUser(): Promise { beforeAll(async () => { sql = postgres(URL, { onnotice: () => {} }); coreSchema = `core_test_${rand(8)}`; + authSchema = `auth_test_${rand(8)}`; await bootstrapSpaceDatabase(sql); // extensions for me_ (space.create) await migrateCore(sql, { schema: coreSchema }); + await migrateAuth(sql, { schema: authSchema }); }); afterAll(async () => { @@ -74,6 +83,7 @@ afterAll(async () => { await sql.unsafe(`drop schema if exists ${s} cascade`); } await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); await sql.end(); }); @@ -81,6 +91,24 @@ beforeEach(async () => { userId = await makeUser(); }); +test("whoami returns the session's identity", async () => { + // a user in the auth schema; whoami resolves it from ctx.auth by id + const email = `who_${rand(8)}@example.com`; + const id = await authStore(sql, authSchema).createUser(email, "Who Am I"); + + const me = await call<{ id: string; email: string; name: string }>( + "whoami", + {}, + id, + ); + expect(me).toEqual({ id, email, name: "Who Am I" }); +}); + +test("whoami is UNAUTHORIZED when the user row is gone", async () => { + const [row] = await sql`select uuidv7() as id`; + await expectAppError(call("whoami", {}, row?.id as string), "UNAUTHORIZED"); +}); + test("create / list / rename / delete the caller's agents", async () => { const { id } = await call<{ id: string }>("agent.create", { name: "bot" }); diff --git a/packages/server/rpc/user/agent.ts b/packages/server/rpc/user/agent.ts index 611e9e0..de5661a 100644 --- a/packages/server/rpc/user/agent.ts +++ b/packages/server/rpc/user/agent.ts @@ -3,8 +3,8 @@ * * Agents are a user's global service accounts. The lifecycle here is purely * user-scoped: create / list / rename / delete the caller's own agents. - * Bringing an agent into a space (member.add) and minting its space-bound api - * key (apiKey.create) are space-endpoint operations. + * Bringing an agent into a space (principal.add) and minting its space-bound + * api key (apiKey.create) are space-endpoint operations. */ import type { Principal } from "@memory.build/engine/core"; import type { diff --git a/packages/server/rpc/user/index.ts b/packages/server/rpc/user/index.ts index 804f633..f7721e6 100644 --- a/packages/server/rpc/user/index.ts +++ b/packages/server/rpc/user/index.ts @@ -5,6 +5,7 @@ import type { MethodRegistry } from "../types"; import { agentMethods } from "./agent"; import { spaceMethods } from "./space"; +import { whoamiMethods } from "./whoami"; export { assertUserRpcContext, @@ -12,8 +13,9 @@ export { type UserRpcContext, } from "./types"; -/** The user-endpoint registry: agent lifecycle + space discovery. */ +/** The user-endpoint registry: identity + agent lifecycle + space discovery. */ export const userMethods: MethodRegistry = new Map([ + ...whoamiMethods, ...agentMethods, ...spaceMethods, ]); diff --git a/packages/server/rpc/user/types.ts b/packages/server/rpc/user/types.ts index 7687797..16a470f 100644 --- a/packages/server/rpc/user/types.ts +++ b/packages/server/rpc/user/types.ts @@ -2,6 +2,7 @@ * User RPC context — populated by authenticateUser. User-scoped (no space): * the calling user manages their own global service accounts (agents). */ +import type { AuthStore } from "@memory.build/auth"; import type { CoreStore } from "@memory.build/engine/core"; import type { Sql } from "postgres"; import type { HandlerContext } from "../types"; @@ -9,6 +10,8 @@ import type { HandlerContext } from "../types"; export interface UserRpcContext extends HandlerContext { /** Core control-plane store. */ core: CoreStore; + /** Auth store (auth schema) — for the caller's identity (whoami). */ + auth: AuthStore; /** The authenticated user id (== the core user-principal id). */ userId: string; /** New-model pool — for transactional provisioning (space.create). */ diff --git a/packages/server/rpc/user/whoami.ts b/packages/server/rpc/user/whoami.ts new file mode 100644 index 0000000..15830ca --- /dev/null +++ b/packages/server/rpc/user/whoami.ts @@ -0,0 +1,27 @@ +/** + * whoami handler for the user RPC — the identity behind the session token. + */ +import type { WhoamiParams, WhoamiResult } from "@memory.build/protocol/user"; +import { whoamiParams } from "@memory.build/protocol/user"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +async function whoami( + _params: WhoamiParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const user = await ctx.auth.getUser(ctx.userId); + if (!user) { + // The session validated but the user row is gone — treat as unauthenticated. + throw new AppError("UNAUTHORIZED", "User not found"); + } + return { id: user.id, email: user.email, name: user.name }; +} + +export const whoamiMethods = buildRegistry() + .register("whoami", whoamiParams, whoami) + .build(); From 48fe97a0c90309df23923d00ca9148619d14dca9 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:40:53 +0200 Subject: [PATCH 053/156] feat(cli): me space command group (4E-CLI-3) New space-model command group on the user/memory clients: - list (ls): your spaces, with admin + active markers - use : set the active space (resolve by slug/name; picker when no arg) - create : create and auto-activate (creator is admin + owner@root) - rename : rename the display label (slug is immutable) - delete (rm): type-name confirmation; clears the active pointer when the deleted space was active - invite [--admin]: principal.resolveByEmail -> principal.add on the active space; clear error when the user hasn't signed in yet Adds buildUserClient / buildMemoryClient helpers (type-guarded on the session token / active space) and clearActiveSpace; wired into index.ts. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/space.ts | 347 +++++++++++++++++++++++++++++++++ packages/cli/credentials.ts | 13 ++ packages/cli/index.ts | 4 + packages/cli/util.ts | 33 +++- 4 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 packages/cli/commands/space.ts diff --git a/packages/cli/commands/space.ts b/packages/cli/commands/space.ts new file mode 100644 index 0000000..969678b --- /dev/null +++ b/packages/cli/commands/space.ts @@ -0,0 +1,347 @@ +/** + * me space — manage the spaces you belong to and the active space. + * + * - me space list: list your spaces (marks the active one) + * - me space use : set the active space (the X-Me-Space) + * - me space create : create a space and make it active + * - me space rename : rename a space's display label + * - me space delete : delete a space and all its data + * - me space invite [--admin]: add an existing user to the active space + * + * accepts a slug (exact) or a name (case-insensitive). The slug is the + * immutable 12-char routing key; the name is the renamable display label. + */ +import * as clack from "@clack/prompts"; +import type { MemberSpaceResponse } from "@memory.build/protocol/user"; +import { Command } from "commander"; +import { createUserClient } from "../client.ts"; +import { + clearActiveSpace, + resolveCredentials, + setActiveSpace, +} from "../credentials.ts"; +import { + getOutputFormat, + type OutputFormat, + output, + table, +} from "../output.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, +} from "../util.ts"; + +/** + * Resolve a argument against the caller's spaces by slug (exact) or + * name (case-insensitive). With no argument, prompts in text mode. Exits on a + * miss / ambiguity / non-interactive-without-arg. + */ +async function resolveSpaceArg( + spaces: MemberSpaceResponse[], + arg: string | undefined, + fmt: OutputFormat, +): Promise { + if (!arg) { + if (fmt !== "text") { + output({ error: "A space slug or name is required" }, fmt, () => {}); + process.exit(1); + } + if (spaces.length === 0) { + clack.log.error("You don't belong to any spaces."); + process.exit(1); + } + const selected = await clack.select({ + message: "Select a space", + options: spaces.map((s) => ({ + value: s.slug, + label: s.name, + hint: s.slug, + })), + }); + if (clack.isCancel(selected)) { + clack.cancel("Cancelled."); + process.exit(0); + } + const picked = spaces.find((s) => s.slug === selected); + if (picked) return picked; + process.exit(1); + } + + const bySlug = spaces.find((s) => s.slug === arg); + if (bySlug) return bySlug; + + const lower = arg.toLowerCase(); + const byName = spaces.filter((s) => s.name.toLowerCase() === lower); + if (byName.length === 1 && byName[0]) return byName[0]; + + if (byName.length === 0) { + const msg = `No space matching '${arg}'.`; + if (fmt === "text") { + clack.log.error(msg); + for (const s of spaces) console.log(` ${s.name} (${s.slug})`); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); + } + + const msg = `Multiple spaces named '${arg}'. Use the slug instead:`; + if (fmt === "text") { + clack.log.error(msg); + for (const s of byName) console.log(` ${s.name} — ${s.slug}`); + } else { + output({ error: msg, matches: byName }, fmt, () => {}); + } + process.exit(1); +} + +function createSpaceListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list the spaces you belong to") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + output( + { + spaces: spaces.map((s) => ({ + ...s, + active: s.slug === creds.activeSpace, + })), + }, + fmt, + () => { + if (spaces.length === 0) { + console.log(" No spaces. Run 'me space create '."); + return; + } + table( + ["name", "slug", "admin", "active"], + spaces.map((s) => [ + s.name, + s.slug, + s.admin ? "yes" : "", + s.slug === creds.activeSpace ? "active" : "", + ]), + ); + }, + ); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceUseCommand(): Command { + return new Command("use") + .description("set the active space") + .argument("[space]", "space slug or name") + .action(async (arg: string | undefined, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + const space = await resolveSpaceArg(spaces, arg, fmt); + setActiveSpace(creds.server, space.slug); + output({ space, switched: true }, fmt, () => { + clack.log.success(`Active space: ${space.name} (${space.slug})`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceCreateCommand(): Command { + return new Command("create") + .description("create a new space and make it active") + .argument("", "space display name") + .action(async (name: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const created = await user.space.create({ name }); + // A new space's creator is its admin + owner@root — make it active. + setActiveSpace(creds.server, created.slug); + output({ ...created, name, active: true }, fmt, () => { + clack.log.success(`Created space '${name}' (${created.slug})`); + clack.log.info("It is now your active space."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceRenameCommand(): Command { + return new Command("rename") + .description("rename a space's display label (the slug is immutable)") + .argument("", "space slug or name") + .argument("", "new display name") + .action(async (arg: string, newName: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + const space = await resolveSpaceArg(spaces, arg, fmt); + const oldName = space.name; + const result = await user.space.rename({ + slug: space.slug, + name: newName, + }); + output({ slug: space.slug, name: newName, ...result }, fmt, () => { + clack.log.success( + `Renamed space '${oldName}' → '${newName}' (${space.slug})`, + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceDeleteCommand(): Command { + return new Command("delete") + .alias("rm") + .description("permanently delete a space and all its data") + .argument("", "space slug or name") + .option("--force", "skip confirmation prompt") + .action(async (arg: string, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + const space = await resolveSpaceArg(spaces, arg, fmt); + + if (fmt === "text" && !opts.force) { + clack.log.warn( + "This permanently deletes the space and ALL its data (memories, grants, groups).", + ); + clack.log.warn("This action cannot be undone."); + const confirmation = await clack.text({ + message: `Type the space name "${space.name}" to confirm deletion`, + validate: (value) => + value !== space.name + ? `Please type "${space.name}" exactly to confirm` + : undefined, + }); + if (clack.isCancel(confirmation)) { + clack.cancel("Cancelled."); + process.exit(0); + } + } + + const result = await user.space.delete({ slug: space.slug }); + // If we just deleted the active space, drop the stale pointer. + if (result.deleted && creds.activeSpace === space.slug) { + clearActiveSpace(creds.server); + } + output({ slug: space.slug, ...result }, fmt, () => { + if (result.deleted) { + clack.log.success(`Space '${space.name}' has been deleted.`); + if (creds.activeSpace === space.slug) { + clack.log.info("Run 'me space use ' to pick another."); + } + } + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceInviteCommand(): Command { + return new Command("invite") + .description("add an existing user to the active space (by email)") + .argument("", "the user's email") + .option("--admin", "grant space-admin (manage members and groups)") + .action(async (email: string, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + + try { + const { principal } = await memory.principal.resolveByEmail({ email }); + if (!principal) { + const msg = `No user found with email '${email}'. They must sign in once before they can be added.`; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); + } + + const result = await memory.principal.add({ + principalId: principal.id, + admin: opts.admin === true, + }); + output({ email, principalId: principal.id, ...result }, fmt, () => { + clack.log.success( + `Added ${email} to the space${opts.admin ? " as an admin" : ""}.`, + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +export function createSpaceCommand(): Command { + const space = new Command("space").description("manage spaces"); + space.addCommand(createSpaceListCommand()); + space.addCommand(createSpaceUseCommand()); + space.addCommand(createSpaceCreateCommand()); + space.addCommand(createSpaceRenameCommand()); + space.addCommand(createSpaceDeleteCommand()); + space.addCommand(createSpaceInviteCommand()); + return space; +} diff --git a/packages/cli/credentials.ts b/packages/cli/credentials.ts index 2166fdd..65973ac 100644 --- a/packages/cli/credentials.ts +++ b/packages/cli/credentials.ts @@ -281,6 +281,19 @@ export function setActiveSpace(server: string, spaceSlug: string): void { writeCredentials(creds); } +/** + * Clear the active space for a server (e.g. after deleting it). No-op if none + * is set. + */ +export function clearActiveSpace(server: string): void { + const creds = readCredentials(); + const origin = normalizeOrigin(server); + const entry = creds.servers[origin]; + if (!entry?.active_space) return; + delete entry.active_space; + writeCredentials(creds); +} + /** * Resolve the active space slug for a server. * diff --git a/packages/cli/index.ts b/packages/cli/index.ts index 3738174..5c19bb0 100755 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -26,6 +26,7 @@ import { createOwnerCommand } from "./commands/owner.ts"; import { createPackCommand } from "./commands/pack.ts"; import { createRoleCommand } from "./commands/role.ts"; import { createServeCommand } from "./commands/serve.ts"; +import { createSpaceCommand } from "./commands/space.ts"; import { createUpgradeCommand } from "./commands/upgrade.ts"; import { createUserCommand } from "./commands/user.ts"; import { createVersionCommand } from "./commands/version.ts"; @@ -67,6 +68,9 @@ program.addCommand(createUpgradeCommand()); // Engine commands program.addCommand(createEngineCommand()); +// Space commands (new model) +program.addCommand(createSpaceCommand()); + // Org commands program.addCommand(createOrgCommand()); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 71418f5..1b006f4 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -8,8 +8,13 @@ * - Error handling */ import * as clack from "@clack/prompts"; -import type { AccountsClient, EngineClient } from "./client.ts"; -import { RpcError } from "./client.ts"; +import type { + AccountsClient, + EngineClient, + MemoryClient, + UserClient, +} from "./client.ts"; +import { createMemoryClient, createUserClient, RpcError } from "./client.ts"; import { clearSessionToken, type ResolvedCredentials } from "./credentials.ts"; import type { OutputFormat } from "./output.ts"; import { output } from "./output.ts"; @@ -71,6 +76,30 @@ export function requireSpace( } } +/** + * Build a user client (session-only, /api/v1/user/rpc). Call requireSession + * first so the token is present. + */ +export function buildUserClient( + creds: ResolvedCredentials & { sessionToken: string }, +): UserClient { + return createUserClient({ url: creds.server, token: creds.sessionToken }); +} + +/** + * Build a memory client (session + active space, /api/v1/memory/rpc). Call + * requireSession and requireSpace first so both are present. + */ +export function buildMemoryClient( + creds: ResolvedCredentials & { sessionToken: string; activeSpace: string }, +): MemoryClient { + return createMemoryClient({ + url: creds.server, + token: creds.sessionToken, + space: creds.activeSpace, + }); +} + interface OrgInfo { id: string; name: string; From 7f461a5b8e2087c309f7151d89f3e517cb556d57 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:45:13 +0200 Subject: [PATCH 054/156] feat(cli): memory commands on memoryClient + top-level aliases (4E-CLI-4) Cut the memory data-plane commands over from the engine api-key client to the memory client (session token + active space via X-Me-Space). - memory.ts: every subcommand uses buildMemoryClient(creds) + requireSpace. The .memory.* methods and their param/result types are identical (both come from @memory.build/protocol/engine/memory), so the handlers are unchanged. - memory-edit.ts: editMemory typed to MemoryClient. - memory-import.ts: buildMemoryClient + requireSpace; batchCreateChunked's structural BatchCreateClient accepts it unchanged. - top-level aliases: memorySubcommands() is shared between the `memory` group and the root program, so `me search` / `me create` / `me get` / ... work as aliases for `me memory `. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/memory-edit.ts | 4 +- packages/cli/commands/memory-import.ts | 7 +- packages/cli/commands/memory.ts | 117 +++++++++++++++---------- packages/cli/index.ts | 8 +- 4 files changed, 81 insertions(+), 55 deletions(-) diff --git a/packages/cli/commands/memory-edit.ts b/packages/cli/commands/memory-edit.ts index 4915d18..ab8046e 100644 --- a/packages/cli/commands/memory-edit.ts +++ b/packages/cli/commands/memory-edit.ts @@ -10,7 +10,7 @@ import { unlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { stringify as yamlStringify } from "yaml"; -import type { EngineClient } from "../client.ts"; +import type { MemoryClient } from "../client.ts"; import { parseMarkdown } from "../parsers/markdown.ts"; interface ParsedMemory { @@ -115,7 +115,7 @@ function hasChanges( * Edit a memory interactively. */ export async function editMemory( - engine: EngineClient, + engine: MemoryClient, id: string, ): Promise { const original = await engine.memory.get({ id }); diff --git a/packages/cli/commands/memory-import.ts b/packages/cli/commands/memory-import.ts index 4b1f935..8a9b115 100644 --- a/packages/cli/commands/memory-import.ts +++ b/packages/cli/commands/memory-import.ts @@ -11,7 +11,6 @@ import { resolve } from "node:path"; import * as clack from "@clack/prompts"; import { Command } from "commander"; import { batchCreateChunked } from "../chunk.ts"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output } from "../output.ts"; import { @@ -19,7 +18,7 @@ import { type ParsedMemory, parseContent, } from "../parsers/index.ts"; -import { requireEngine, requireSession } from "../util.ts"; +import { buildMemoryClient, requireSession, requireSpace } from "../util.ts"; /** * Collect files from a path. If directory, requires --recursive. @@ -103,7 +102,7 @@ export function createMemoryImportCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); // Validate format option if (opts.format && !["md", "yaml", "json"].includes(opts.format)) { @@ -265,7 +264,7 @@ export function createMemoryImportCommand(): Command { } // Actual import - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const engine = buildMemoryClient(creds); let skippedIds: string[] = []; diff --git a/packages/cli/commands/memory.ts b/packages/cli/commands/memory.ts index 59fc5f3..2107fbe 100644 --- a/packages/cli/commands/memory.ts +++ b/packages/cli/commands/memory.ts @@ -18,10 +18,14 @@ import { join } from "node:path"; import * as clack from "@clack/prompts"; import { Command } from "commander"; import { stringify as yamlStringify } from "yaml"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; -import { handleError, requireEngine, requireSession } from "../util.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, +} from "../util.ts"; import { editMemory } from "./memory-edit.ts"; import { createMemoryImportCommand } from "./memory-import.ts"; import { renderTree } from "./memory-tree.ts"; @@ -104,7 +108,7 @@ function createMemoryCreateCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); // Resolve content: positional > --content flag > stdin let content = positionalContent ?? opts.content; @@ -127,7 +131,7 @@ function createMemoryCreateCommand(): Command { process.exit(1); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { const params: Record = { content }; @@ -135,8 +139,8 @@ function createMemoryCreateCommand(): Command { if (opts.meta) params.meta = parseMeta(opts.meta); if (opts.temporal) params.temporal = parseTemporal(opts.temporal); - const memory = await engine.memory.create( - params as Parameters[0], + const memory = await client.memory.create( + params as Parameters[0], ); output(memory, fmt, () => { @@ -159,12 +163,12 @@ function createMemoryGetCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { - const memory = await engine.memory.get({ id }); + const memory = await client.memory.get({ id }); // --json / --yaml: structured output if (fmt !== "text") { @@ -232,7 +236,7 @@ function createMemorySearchCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); // Resolve search text. A positional query runs hybrid search if (query && opts.semantic && opts.fulltext) { @@ -304,7 +308,7 @@ function createMemorySearchCommand(): Command { weights.fulltext = Number.parseFloat(opts.weightFulltext); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { const params: Record = { @@ -323,8 +327,8 @@ function createMemorySearchCommand(): Command { params.semanticThreshold = Number.parseFloat(opts.semanticThreshold); if (opts.orderBy) params.orderBy = opts.orderBy; - const result = await engine.memory.search( - params as Parameters[0], + const result = await client.memory.search( + params as Parameters[0], ); output(result, fmt, () => { @@ -368,7 +372,7 @@ function createMemoryUpdateCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); // Resolve content let content = opts.content; @@ -387,7 +391,7 @@ function createMemoryUpdateCommand(): Command { process.exit(1); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { const params: Record = { id }; @@ -396,8 +400,8 @@ function createMemoryUpdateCommand(): Command { if (opts.meta) params.meta = parseMeta(opts.meta); if (opts.temporal) params.temporal = parseTemporal(opts.temporal); - const memory = await engine.memory.update( - params as Parameters[0], + const memory = await client.memory.update( + params as Parameters[0], ); output(memory, fmt, () => { @@ -421,14 +425,14 @@ function createMemoryDeleteCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { if (UUIDV7_RE.test(idOrTree)) { // Single memory delete - const result = await engine.memory.delete({ id: idOrTree }); + const result = await client.memory.delete({ id: idOrTree }); output(result, fmt, () => { if (result.deleted) { clack.log.success(`Deleted memory ${idOrTree}`); @@ -438,7 +442,7 @@ function createMemoryDeleteCommand(): Command { }); } else { // Tree delete — always dry-run first - const preview = await engine.memory.deleteTree({ + const preview = await client.memory.deleteTree({ tree: idOrTree, dryRun: true, }); @@ -473,7 +477,7 @@ function createMemoryDeleteCommand(): Command { } } - const result = await engine.memory.deleteTree({ + const result = await client.memory.deleteTree({ tree: idOrTree, dryRun: false, }); @@ -498,12 +502,12 @@ function createMemoryEditCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { - await editMemory(engine, id); + await editMemory(client, id); } catch (error) { handleError(error, fmt); } @@ -520,17 +524,17 @@ function createMemoryTreeCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { const params: Record = {}; if (filter) params.tree = filter; if (opts.levels) params.levels = Number.parseInt(opts.levels, 10); - const result = await engine.memory.tree( - params as Parameters[0], + const result = await client.memory.tree( + params as Parameters[0], ); output(result, fmt, () => { @@ -555,13 +559,13 @@ function createMemoryMoveCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { // Always dry-run first to show preview - const preview = await engine.memory.move({ + const preview = await client.memory.move({ source: src, destination: dst, dryRun: true, @@ -597,7 +601,7 @@ function createMemoryMoveCommand(): Command { } } - const result = await engine.memory.move({ + const result = await client.memory.move({ source: src, destination: dst, }); @@ -651,7 +655,7 @@ function createMemoryExportCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); const format = opts.format as "json" | "yaml" | "md"; if (!["json", "yaml", "md"].includes(format)) { @@ -694,11 +698,11 @@ function createMemoryExportCommand(): Command { }; } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { - const result = await engine.memory.search( - searchParams as Parameters[0], + const result = await client.memory.search( + searchParams as Parameters[0], ); const memories = result.results.map((r: Record) => @@ -871,17 +875,36 @@ function renderMarkdownAnsi(content: string): string { // Command Group // ============================================================================= +/** + * Build a fresh set of the memory subcommands. A Commander command can only be + * attached to one parent, so callers that want them in two places (the `memory` + * group and the top-level aliases) each call this for their own instances. + */ +function memorySubcommands(): Command[] { + return [ + createMemoryCreateCommand(), + createMemoryGetCommand(), + createMemorySearchCommand(), + createMemoryUpdateCommand(), + createMemoryDeleteCommand(), + createMemoryEditCommand(), + createMemoryTreeCommand(), + createMemoryMoveCommand(), + createMemoryImportCommand(), + createMemoryExportCommand(), + ]; +} + export function createMemoryCommand(): Command { const memory = new Command("memory").description("manage memories"); - memory.addCommand(createMemoryCreateCommand()); - memory.addCommand(createMemoryGetCommand()); - memory.addCommand(createMemorySearchCommand()); - memory.addCommand(createMemoryUpdateCommand()); - memory.addCommand(createMemoryDeleteCommand()); - memory.addCommand(createMemoryEditCommand()); - memory.addCommand(createMemoryTreeCommand()); - memory.addCommand(createMemoryMoveCommand()); - memory.addCommand(createMemoryImportCommand()); - memory.addCommand(createMemoryExportCommand()); + for (const c of memorySubcommands()) memory.addCommand(c); return memory; } + +/** + * The memory subcommands as top-level aliases (`me search`, `me create`, …) so + * the `memory` word is optional for the common data-plane operations. + */ +export function createMemoryAliasCommands(): Command[] { + return memorySubcommands(); +} diff --git a/packages/cli/index.ts b/packages/cli/index.ts index 5c19bb0..99a6d5d 100755 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -19,7 +19,10 @@ import { createInvitationCommand } from "./commands/invitation.ts"; import { createLoginCommand } from "./commands/login.ts"; import { createLogoutCommand } from "./commands/logout.ts"; import { createMcpCommand } from "./commands/mcp.ts"; -import { createMemoryCommand } from "./commands/memory.ts"; +import { + createMemoryAliasCommands, + createMemoryCommand, +} from "./commands/memory.ts"; import { createOpenCodeCommand } from "./commands/opencode.ts"; import { createOrgCommand } from "./commands/org.ts"; import { createOwnerCommand } from "./commands/owner.ts"; @@ -77,8 +80,9 @@ program.addCommand(createOrgCommand()); // Invitation commands program.addCommand(createInvitationCommand()); -// Memory commands +// Memory commands — both as `me memory ` and top-level aliases (`me search`) program.addCommand(createMemoryCommand()); +for (const c of createMemoryAliasCommands()) program.addCommand(c); // MCP server program.addCommand(createMcpCommand()); From 0e5f314344d95d1b1cbbf8a8d8386cabb906e1ca Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:48:32 +0200 Subject: [PATCH 055/156] feat(cli): me group + me access command groups (4E-CLI-5a) Space-scoped management commands on the memory client (session + active space). - me access grant/rm-grant/list: 3-level tree grants (r=read, w=write, o=owner) keyed by principal + tree path; list maps principal ids to names best-effort and shows the access level by name. - me group list/create/rename/delete/add/remove/members: manage groups and their user/agent members. Adds resolveSpacePrincipalId (UUID or name; user=email, agent/group=display name) to util.ts; wired both groups into index.ts. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/access.ts | 194 ++++++++++++++++++++++ packages/cli/commands/group.ts | 278 ++++++++++++++++++++++++++++++++ packages/cli/index.ts | 4 + packages/cli/util.ts | 41 +++++ 4 files changed, 517 insertions(+) create mode 100644 packages/cli/commands/access.ts create mode 100644 packages/cli/commands/group.ts diff --git a/packages/cli/commands/access.ts b/packages/cli/commands/access.ts new file mode 100644 index 0000000..8fc2ff1 --- /dev/null +++ b/packages/cli/commands/access.ts @@ -0,0 +1,194 @@ +/** + * me access — tree-access grants in the active space. + * + * The core model uses three additive levels: r = read, w = write, o = owner. + * Grants are keyed by (principal, tree path); an owner grant at a path lets the + * holder manage access within that subtree. + * + * - me access grant : grant or update access + * - me access rm-grant : remove a grant + * - me access list [principal] [--path

]: list grants (optionally scoped) + * + * is a UUID, or a name (user = email, agent/group = display name). + */ +import * as clack from "@clack/prompts"; +import { Command } from "commander"; +import { resolveCredentials } from "../credentials.ts"; +import { getOutputFormat, output, table } from "../output.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, + resolveSpacePrincipalId, +} from "../util.ts"; + +/** Access level: r = read (1), w = write (2), o = owner (3). */ +const LEVELS: Record = { + r: 1, + read: 1, + w: 2, + write: 2, + o: 3, + owner: 3, +}; +const LEVEL_NAME: Record = { + 1: "read", + 2: "write", + 3: "owner", +}; + +function parseLevel(input: string): 1 | 2 | 3 | null { + return LEVELS[input.toLowerCase()] ?? null; +} + +function createAccessGrantCommand(): Command { + return new Command("grant") + .description("grant or update a principal's access at a tree path") + .argument( + "", + "principal id or name (user email / agent / group)", + ) + .argument("", "tree path (empty string for the space root)") + .argument("", "access level: r (read), w (write), o (owner)") + .action( + async (principal: string, path: string, level: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const access = parseLevel(level); + if (!access) { + handleError( + new Error(`Invalid level '${level}'. Use r, w, or o.`), + fmt, + ); + } + + const memory = buildMemoryClient(creds); + try { + const principalId = await resolveSpacePrincipalId( + memory, + principal, + fmt, + ); + const result = await memory.grant.set({ + principalId, + treePath: path, + access, + }); + output( + { principalId, treePath: path, access, ...result }, + fmt, + () => { + clack.log.success( + `Granted ${LEVEL_NAME[access]} on '${path}' to ${principal}`, + ); + }, + ); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }, + ); +} + +function createAccessRmGrantCommand(): Command { + return new Command("rm-grant") + .description("remove a principal's grant at a tree path") + .argument("", "principal id or name") + .argument("", "tree path") + .action(async (principal: string, path: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const principalId = await resolveSpacePrincipalId( + memory, + principal, + fmt, + ); + const result = await memory.grant.remove({ + principalId, + treePath: path, + }); + output({ principalId, treePath: path, ...result }, fmt, () => { + if (result.removed) { + clack.log.success(`Removed grant on '${path}' from ${principal}`); + } else { + clack.log.warn("Grant not found."); + } + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAccessListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list grants in the active space") + .argument("[principal]", "filter by principal id or name") + .option("--path ", "only grants at or below this tree path") + .action(async (principal: string | undefined, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const principalId = principal + ? await resolveSpacePrincipalId(memory, principal, fmt) + : undefined; + const { grants } = await memory.grant.list({ + principalId: principalId ?? null, + treePath: opts.path ?? null, + }); + + // Map principal ids → names for display (best-effort). + const names = new Map(); + try { + const { principals } = await memory.principal.list({}); + for (const p of principals) names.set(p.id, p.name); + } catch { + // listing principals may require more authority than listing grants + } + + output({ grants }, fmt, () => { + if (grants.length === 0) { + console.log(" No grants found."); + return; + } + table( + ["principal", "tree_path", "access"], + grants.map((g) => [ + names.get(g.principalId) ?? g.principalId, + g.treePath === "" ? "(root)" : g.treePath, + LEVEL_NAME[g.access] ?? String(g.access), + ]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +export function createAccessCommand(): Command { + const access = new Command("access").description( + "manage tree-access grants in the active space", + ); + access.addCommand(createAccessGrantCommand()); + access.addCommand(createAccessRmGrantCommand()); + access.addCommand(createAccessListCommand()); + return access; +} diff --git a/packages/cli/commands/group.ts b/packages/cli/commands/group.ts new file mode 100644 index 0000000..1010b70 --- /dev/null +++ b/packages/cli/commands/group.ts @@ -0,0 +1,278 @@ +/** + * me group — manage groups in the active space. + * + * Groups bundle members (users / agents) so a single tree-access grant covers + * everyone in the group. Group membership also confers space membership. + * + * - me group list: list groups + * - me group create : create a group + * - me group rename : rename a group + * - me group delete : delete a group + * - me group add [--admin]: add a member (user/agent) + * - me group remove : remove a member + * - me group members : list a group's members + * + * is a group id or name; is a user/agent id or name (a UUID is + * always accepted; name resolution requires space-manager authority). + */ +import * as clack from "@clack/prompts"; +import { Command } from "commander"; +import type { MemoryClient } from "../client.ts"; +import { resolveCredentials } from "../credentials.ts"; +import { + getOutputFormat, + type OutputFormat, + output, + table, +} from "../output.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, + resolveSpacePrincipalId, +} from "../util.ts"; + +const UUIDV7_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +/** Resolve a group id from a UUID or name (via group.list). */ +async function resolveGroupId( + memory: MemoryClient, + input: string, + fmt: OutputFormat, +): Promise { + if (UUIDV7_RE.test(input)) return input; + const { groups } = await memory.group.list(); + const lower = input.toLowerCase(); + const matches = groups.filter((g) => g.name.toLowerCase() === lower); + if (matches.length === 1 && matches[0]) return matches[0].id; + const msg = + matches.length === 0 + ? `No group named '${input}' in this space.` + : `Multiple groups named '${input}'. Use the group id instead.`; + if (fmt === "text") { + clack.log.error(msg); + if (matches.length > 1) + for (const g of matches) console.log(` ${g.name} — ${g.id}`); + } else { + output({ error: msg, matches }, fmt, () => {}); + } + process.exit(1); +} + +function createGroupListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list groups in the active space") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const { groups } = await memory.group.list(); + output({ groups }, fmt, () => { + if (groups.length === 0) { + console.log(" No groups. Run 'me group create '."); + return; + } + table( + ["name", "id"], + groups.map((g) => [g.name, g.id]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupCreateCommand(): Command { + return new Command("create") + .description("create a group") + .argument("", "group name") + .action(async (name: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const { id } = await memory.group.create({ name }); + output({ id, name }, fmt, () => { + clack.log.success(`Created group '${name}' (${id})`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupRenameCommand(): Command { + return new Command("rename") + .description("rename a group") + .argument("", "group id or name") + .argument("", "new name") + .action(async (group: string, newName: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const id = await resolveGroupId(memory, group, fmt); + const result = await memory.group.rename({ id, name: newName }); + output({ id, name: newName, ...result }, fmt, () => { + clack.log.success(`Renamed group → '${newName}'`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupDeleteCommand(): Command { + return new Command("delete") + .alias("rm") + .description("delete a group") + .argument("", "group id or name") + .action(async (group: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const id = await resolveGroupId(memory, group, fmt); + const result = await memory.group.delete({ id }); + output({ id, ...result }, fmt, () => { + if (result.deleted) clack.log.success(`Deleted group ${group}`); + else clack.log.warn("Group not found."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupAddCommand(): Command { + return new Command("add") + .description("add a member (user/agent) to a group") + .argument("", "group id or name") + .argument("", "user/agent id or name") + .option("--admin", "make them a group admin (can manage group membership)") + .action(async (group: string, member: string, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const groupId = await resolveGroupId(memory, group, fmt); + const memberId = await resolveSpacePrincipalId(memory, member, fmt); + const result = await memory.group.addMember({ + groupId, + memberId, + admin: opts.admin === true, + }); + output({ groupId, memberId, ...result }, fmt, () => { + clack.log.success( + `Added ${member} to group ${group}${opts.admin ? " as an admin" : ""}.`, + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupRemoveCommand(): Command { + return new Command("remove") + .alias("rm-member") + .description("remove a member from a group") + .argument("", "group id or name") + .argument("", "user/agent id or name") + .action(async (group: string, member: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const groupId = await resolveGroupId(memory, group, fmt); + const memberId = await resolveSpacePrincipalId(memory, member, fmt); + const result = await memory.group.removeMember({ groupId, memberId }); + output({ groupId, memberId, ...result }, fmt, () => { + if (result.removed) + clack.log.success(`Removed ${member} from ${group}`); + else clack.log.warn("Member not in group."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupMembersCommand(): Command { + return new Command("members") + .description("list a group's members") + .argument("", "group id or name") + .action(async (group: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const groupId = await resolveGroupId(memory, group, fmt); + const { members } = await memory.group.listMembers({ groupId }); + output({ members }, fmt, () => { + if (members.length === 0) { + console.log(" No members."); + return; + } + table( + ["name", "kind", "admin", "id"], + members.map((m) => [ + m.name, + m.kind, + m.admin ? "yes" : "", + m.memberId, + ]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +export function createGroupCommand(): Command { + const group = new Command("group").description( + "manage groups in the active space", + ); + group.addCommand(createGroupListCommand()); + group.addCommand(createGroupCreateCommand()); + group.addCommand(createGroupRenameCommand()); + group.addCommand(createGroupDeleteCommand()); + group.addCommand(createGroupAddCommand()); + group.addCommand(createGroupRemoveCommand()); + group.addCommand(createGroupMembersCommand()); + return group; +} diff --git a/packages/cli/index.ts b/packages/cli/index.ts index 99a6d5d..ad6056c 100755 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -9,12 +9,14 @@ import { Command } from "commander"; * and all command groups, then runs. */ import { CLIENT_VERSION } from "../../version"; +import { createAccessCommand } from "./commands/access.ts"; import { createApiKeyCommand } from "./commands/apikey.ts"; import { createClaudeCommand } from "./commands/claude.ts"; import { createCodexCommand } from "./commands/codex.ts"; import { createEngineCommand } from "./commands/engine.ts"; import { createGeminiCommand } from "./commands/gemini.ts"; import { createGrantCommand } from "./commands/grant.ts"; +import { createGroupCommand } from "./commands/group.ts"; import { createInvitationCommand } from "./commands/invitation.ts"; import { createLoginCommand } from "./commands/login.ts"; import { createLogoutCommand } from "./commands/logout.ts"; @@ -73,6 +75,8 @@ program.addCommand(createEngineCommand()); // Space commands (new model) program.addCommand(createSpaceCommand()); +program.addCommand(createGroupCommand()); +program.addCommand(createAccessCommand()); // Org commands program.addCommand(createOrgCommand()); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 1b006f4..41e7316 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -100,6 +100,47 @@ export function buildMemoryClient( }); } +/** + * Resolve a principal in the active space to its id. Accepts a UUIDv7 (used + * as-is) or a name — for users the name is their email; for agents/groups it is + * the display name. Optionally constrained to a kind ('u' | 'a' | 'g'). Listing + * principals requires space-manager authority; callers without it should pass a + * UUID. Exits with an actionable error on miss / ambiguity. + */ +export async function resolveSpacePrincipalId( + memory: MemoryClient, + input: string, + fmt: OutputFormat, + kind?: "u" | "a" | "g", +): Promise { + if (UUIDV7_RE.test(input)) return input; + + const { principals } = await memory.principal.list(kind ? { kind } : {}); + const lower = input.toLowerCase(); + const matches = principals.filter((p) => p.name.toLowerCase() === lower); + + if (matches.length === 1 && matches[0]) return matches[0].id; + + if (matches.length === 0) { + const msg = `No ${kind === "g" ? "group" : "principal"} named '${input}' in this space.`; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); + } + + const msg = `Multiple principals named '${input}'. Use the id instead:`; + if (fmt === "text") { + clack.log.error(msg); + for (const m of matches) console.log(` ${m.name} (${m.kind}) — ${m.id}`); + } else { + output({ error: msg, matches }, fmt, () => {}); + } + process.exit(1); +} + interface OrgInfo { id: string; name: string; From 1be0b88f2fd5882df30442233429f588b7355705 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:51:28 +0200 Subject: [PATCH 056/156] feat(cli): me agent + new-model me apikey (4E-CLI-5b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - me agent (user endpoint): list/create/rename/delete your agents, plus `add ` (bring your own agent into the active space — self-service principal.add) and `groups ` (its groups via group.listForMember). - me apikey (memory endpoint, space-scoped): create/list/get/delete an agent's keys. The key is shown once on create with guidance to hand it to the agent via ME_API_KEY or its MCP config; delete is the revoke (no soft state). The legacy engine-model apikey command is replaced. Adds resolveAgentId (UUID or name) to util.ts; apikey moves to the new-model command group in index.ts. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/agent.ts | 202 +++++++++++++++++++++++++++++ packages/cli/commands/apikey.ts | 217 +++++++++++++------------------- packages/cli/index.ts | 6 +- packages/cli/util.ts | 29 +++++ 4 files changed, 324 insertions(+), 130 deletions(-) create mode 100644 packages/cli/commands/agent.ts diff --git a/packages/cli/commands/agent.ts b/packages/cli/commands/agent.ts new file mode 100644 index 0000000..cc34984 --- /dev/null +++ b/packages/cli/commands/agent.ts @@ -0,0 +1,202 @@ +/** + * me agent — manage your agents (global service accounts). + * + * Agents are owned by you and live across spaces; their lifecycle is on the + * user endpoint. Bringing an agent into the active space and minting its + * (space-bound) api key are space operations — see `me agent add` and + * `me apikey create`. + * + * - me agent list: list your agents + * - me agent create : create an agent + * - me agent rename : rename an agent + * - me agent delete : delete an agent + * - me agent add : add the agent to the active space + * - me agent groups : list the agent's groups in the active space + * + * is an agent id or name (names are unique per user). + */ +import * as clack from "@clack/prompts"; +import { Command } from "commander"; +import { resolveCredentials } from "../credentials.ts"; +import { getOutputFormat, output, table } from "../output.ts"; +import { + buildMemoryClient, + buildUserClient, + handleError, + requireSession, + requireSpace, + resolveAgentId, +} from "../util.ts"; + +function createAgentListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list your agents") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const { agents } = await user.agent.list(); + output({ agents }, fmt, () => { + if (agents.length === 0) { + console.log(" No agents. Run 'me agent create '."); + return; + } + table( + ["name", "id"], + agents.map((a) => [a.name, a.id]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentCreateCommand(): Command { + return new Command("create") + .description("create an agent") + .argument("", "agent name (unique per user)") + .action(async (name: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const { id } = await user.agent.create({ name }); + output({ id, name }, fmt, () => { + clack.log.success(`Created agent '${name}' (${id})`); + clack.log.info( + "Add it to a space with 'me agent add', then mint a key with 'me apikey create'.", + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentRenameCommand(): Command { + return new Command("rename") + .description("rename an agent") + .argument("", "agent id or name") + .argument("", "new name") + .action(async (agent: string, newName: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + const result = await user.agent.rename({ id, name: newName }); + output({ id, name: newName, ...result }, fmt, () => { + clack.log.success(`Renamed agent → '${newName}'`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentDeleteCommand(): Command { + return new Command("delete") + .alias("rm") + .description("delete an agent") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + const result = await user.agent.delete({ id }); + output({ id, ...result }, fmt, () => { + if (result.deleted) clack.log.success(`Deleted agent ${agent}`); + else clack.log.warn("Agent not found."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentAddCommand(): Command { + return new Command("add") + .description("add one of your agents to the active space") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + // Bringing your own agent into a space is self-service (no admin flag). + const result = await memory.principal.add({ principalId: id }); + output({ agentId: id, ...result }, fmt, () => { + clack.log.success(`Added agent ${agent} to the space.`); + clack.log.info("Mint a key with 'me apikey create'."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentGroupsCommand(): Command { + return new Command("groups") + .description("list an agent's groups in the active space") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + const { groups } = await memory.group.listForMember({ memberId: id }); + output({ groups }, fmt, () => { + if (groups.length === 0) { + console.log(" Not in any groups."); + return; + } + table( + ["group", "admin", "id"], + groups.map((g) => [g.name, g.admin ? "yes" : "", g.groupId]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +export function createAgentCommand(): Command { + const agent = new Command("agent").description("manage your agents"); + agent.addCommand(createAgentListCommand()); + agent.addCommand(createAgentCreateCommand()); + agent.addCommand(createAgentRenameCommand()); + agent.addCommand(createAgentDeleteCommand()); + agent.addCommand(createAgentAddCommand()); + agent.addCommand(createAgentGroupsCommand()); + return agent; +} diff --git a/packages/cli/commands/apikey.ts b/packages/cli/commands/apikey.ts index eb83b18..b8a0d2d 100644 --- a/packages/cli/commands/apikey.ts +++ b/packages/cli/commands/apikey.ts @@ -1,167 +1,131 @@ /** - * me apikey — API key management commands. + * me apikey — manage an agent's API keys in the active space. * - * - me apikey list : List API keys for a user - * - me apikey create [name]: Create a new API key - * - me apikey show: Show the API key stored in credentials.yaml - * - me apikey revoke : Revoke an API key - * - me apikey delete : Permanently delete an API key + * Keys are agent-only (humans authenticate with a session). The agent must + * already be in the space (see `me agent add`). The plaintext key is shown + * exactly once, by `create`. There is no revoke state — delete is the removal. + * + * - me apikey create [name] [--expires ]: mint a key (shown once) + * - me apikey list : list an agent's keys + * - me apikey get : key metadata + * - me apikey delete : delete (revoke) a key + * + * is an agent id or name; is an api-key id. */ import * as clack from "@clack/prompts"; import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { - getServerCredentials, - resolveCredentials, - resolveServer, -} from "../credentials.ts"; +import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; import { + buildMemoryClient, + buildUserClient, handleError, - requireEngine, requireSession, - resolveUserId, + requireSpace, + resolveAgentId, } from "../util.ts"; -function createApiKeyListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list API keys for a user") - .argument("", "user name or ID") - .action(async (user: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const { apiKeys } = await engine.apiKey.list({ userId }); - - output({ apiKeys }, fmt, () => { - if (apiKeys.length === 0) { - console.log(" No API keys found."); - return; - } - table( - ["id", "name", "status"], - apiKeys.map((k) => [ - k.id, - k.name, - k.revokedAt ? "revoked" : "active", - ]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - function createApiKeyCreateCommand(): Command { return new Command("create") - .description("create a new API key") - .argument("", "user name or ID") + .description("mint an API key for an agent in the active space") + .argument("", "agent id or name") .argument("[name]", "key name (auto-generated if omitted)") .option("--expires ", "expiration timestamp (ISO 8601)") - .action(async (user: string, name: string | undefined, opts, cmd) => { + .action(async (agent: string, name: string | undefined, opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); + requireSpace(creds, fmt); + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); const keyName = name ?? `cli-${new Date().toISOString().slice(0, 10)}`; try { - const userId = await resolveUserId(engine, user); - const result = await engine.apiKey.create({ - userId, + const agentId = await resolveAgentId(user, agent, fmt); + const result = await memory.apiKey.create({ + agentId, name: keyName, - expiresAt: opts.expires ?? undefined, + expiresAt: opts.expires ?? null, }); - output(result, fmt, () => { - clack.log.success(`Created API key '${result.apiKey.name}'`); - console.log(` ID: ${result.apiKey.id}`); + clack.log.success(`Created API key '${keyName}'`); + console.log(` ID: ${result.id}`); clack.note( - result.rawKey, - "API Key (save this — it won't be shown again)", + result.key, + "API key — save it now; it won't be shown again", + ); + clack.log.info( + "Give it to the agent via ME_API_KEY or its MCP config.", ); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } -function createApiKeyShowCommand(): Command { - return new Command("show") - .description("show the API key stored in credentials.yaml for an engine") - .option("--engine ", "engine slug (defaults to the active engine)") - .action(async (opts, cmd) => { +function createApiKeyListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list an agent's API keys") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); try { - const server = resolveServer(globalOpts.server); - const stored = getServerCredentials(server); - const engine: string | undefined = opts.engine ?? stored.active_engine; - if (!engine) { - throw new Error( - "no active engine for this server. Run 'me engine use ' or pass --engine .", - ); - } - const apiKey = stored.engines?.[engine]?.api_key; - if (!apiKey) { - throw new Error( - `no API key stored for engine '${engine}' on ${server}.`, + const agentId = await resolveAgentId(user, agent, fmt); + const { apiKeys } = await memory.apiKey.list({ memberId: agentId }); + output({ apiKeys }, fmt, () => { + if (apiKeys.length === 0) { + console.log(" No API keys."); + return; + } + table( + ["id", "name", "created", "expires"], + apiKeys.map((k) => [k.id, k.name, k.createdAt, k.expiresAt ?? ""]), ); - } - - output({ server, engine, apiKey }, fmt, () => { - console.log(` Server: ${server}`); - console.log(` Engine: ${engine}`); - console.log(` API key: ${apiKey}`); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } -function createApiKeyRevokeCommand(): Command { - return new Command("revoke") - .description("revoke an API key") - .argument("", "API key ID") +function createApiKeyGetCommand(): Command { + return new Command("get") + .description("show API key metadata") + .argument("", "API key id") .action(async (id: string, _opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + requireSpace(creds, fmt); + const memory = buildMemoryClient(creds); try { - const result = await engine.apiKey.revoke({ id }); - - output(result, fmt, () => { - if (result.revoked) { - clack.log.success("API key revoked."); - } else { + const { apiKey } = await memory.apiKey.get({ id }); + output({ apiKey }, fmt, () => { + if (!apiKey) { clack.log.warn("API key not found."); + return; } + console.log(` ID: ${apiKey.id}`); + console.log(` Name: ${apiKey.name}`); + console.log(` Agent: ${apiKey.memberId}`); + console.log(` Created: ${apiKey.createdAt}`); + console.log(` Expires: ${apiKey.expiresAt ?? "(never)"}`); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } @@ -169,19 +133,20 @@ function createApiKeyRevokeCommand(): Command { function createApiKeyDeleteCommand(): Command { return new Command("delete") .alias("rm") - .description("permanently delete an API key") - .argument("", "API key ID") + .description("delete (revoke) an API key") + .argument("", "API key id") .option("-y, --yes", "skip confirmation prompt") .action(async (id: string, opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); if (fmt === "text" && !opts.yes) { const confirmed = await clack.confirm({ - message: `Permanently delete API key ${id}?`, + message: `Delete API key ${id}? This revokes it immediately.`, + initialValue: false, }); if (clack.isCancel(confirmed) || !confirmed) { clack.cancel("Cancelled."); @@ -189,30 +154,26 @@ function createApiKeyDeleteCommand(): Command { } } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - + const memory = buildMemoryClient(creds); try { - const result = await engine.apiKey.delete({ id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success("API key deleted."); - } else { - clack.log.warn("API key not found."); - } + const result = await memory.apiKey.delete({ id }); + output({ id, ...result }, fmt, () => { + if (result.deleted) clack.log.success("API key deleted."); + else clack.log.warn("API key not found."); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } export function createApiKeyCommand(): Command { - const apikey = new Command("apikey").description("manage API keys"); - apikey.addCommand(createApiKeyListCommand()); + const apikey = new Command("apikey").description( + "manage agent API keys in the active space", + ); apikey.addCommand(createApiKeyCreateCommand()); - apikey.addCommand(createApiKeyShowCommand()); - apikey.addCommand(createApiKeyRevokeCommand()); + apikey.addCommand(createApiKeyListCommand()); + apikey.addCommand(createApiKeyGetCommand()); apikey.addCommand(createApiKeyDeleteCommand()); return apikey; } diff --git a/packages/cli/index.ts b/packages/cli/index.ts index ad6056c..5fbfc3d 100755 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -10,6 +10,7 @@ import { Command } from "commander"; */ import { CLIENT_VERSION } from "../../version"; import { createAccessCommand } from "./commands/access.ts"; +import { createAgentCommand } from "./commands/agent.ts"; import { createApiKeyCommand } from "./commands/apikey.ts"; import { createClaudeCommand } from "./commands/claude.ts"; import { createCodexCommand } from "./commands/codex.ts"; @@ -77,6 +78,8 @@ program.addCommand(createEngineCommand()); program.addCommand(createSpaceCommand()); program.addCommand(createGroupCommand()); program.addCommand(createAccessCommand()); +program.addCommand(createAgentCommand()); +program.addCommand(createApiKeyCommand()); // Org commands program.addCommand(createOrgCommand()); @@ -100,12 +103,11 @@ program.addCommand(createCodexCommand()); // Local web UI program.addCommand(createServeCommand()); -// Engine-level RBAC commands +// Engine-level RBAC commands (legacy; removed in 4E-CLI-7) program.addCommand(createUserCommand()); program.addCommand(createGrantCommand()); program.addCommand(createRoleCommand()); program.addCommand(createOwnerCommand()); -program.addCommand(createApiKeyCommand()); // Pack commands program.addCommand(createPackCommand()); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 41e7316..2a0df46 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -141,6 +141,35 @@ export async function resolveSpacePrincipalId( process.exit(1); } +/** + * Resolve one of the caller's agents to its id, by UUIDv7 or name (agent names + * are unique per user). Exits with an actionable error on miss / ambiguity. + */ +export async function resolveAgentId( + user: UserClient, + input: string, + fmt: OutputFormat, +): Promise { + if (UUIDV7_RE.test(input)) return input; + const { agents } = await user.agent.list(); + const lower = input.toLowerCase(); + const matches = agents.filter((a) => a.name.toLowerCase() === lower); + if (matches.length === 1 && matches[0]) return matches[0].id; + + const msg = + matches.length === 0 + ? `No agent named '${input}'. Run 'me agent list'.` + : `Multiple agents named '${input}'. Use the agent id instead.`; + if (fmt === "text") { + clack.log.error(msg); + if (matches.length > 1) + for (const a of matches) console.log(` ${a.name} — ${a.id}`); + } else { + output({ error: msg, matches }, fmt, () => {}); + } + process.exit(1); +} + interface OrgInfo { id: string; name: string; From 1a78e496b0b6bd09e5dc8b6c87c46e4f64c0d6e4 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 17:54:09 +0200 Subject: [PATCH 057/156] feat(cli): me mcp on the memory client (session or key + space) (4E-CLI-6) The MCP server now talks to /api/v1/memory/rpc instead of the engine api-key client. It only ever used the .memory.* namespace, which is identical on the memory client, so the tool handlers are unchanged. - runMcpServer takes { server, token, space } and builds a memory client with the active space as X-Me-Space. - me mcp resolves token (--api-key > ME_API_KEY > stored session) and space (--space > ME_SPACE / stored active space > the api key's own slug), so a logged-in human gets `me mcp` for free and an agent passes ME_API_KEY (the key embeds its space). - agent-install: api key required for baked MCP configs (a session expires); message points at `me apikey create` instead of the removed `me engine use`. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/mcp.ts | 62 +++++++++++++++++++------------ packages/cli/mcp/agent-install.ts | 6 ++- packages/cli/mcp/server.ts | 18 ++++++--- 3 files changed, 55 insertions(+), 31 deletions(-) diff --git a/packages/cli/commands/mcp.ts b/packages/cli/commands/mcp.ts index df76f50..7561b16 100644 --- a/packages/cli/commands/mcp.ts +++ b/packages/cli/commands/mcp.ts @@ -1,54 +1,68 @@ /** * me mcp — run the MCP server over stdio. * - * Does NOT use the credentials file. API key must be provided - * via --api-key or ME_API_KEY env var. + * Authenticates to a space with either a human session (from `me login`) or an + * agent api key, and targets the active space (the X-Me-Space). Resolution: + * - token: --api-key > ME_API_KEY > stored session token + * - space: --space > ME_SPACE > stored active space > the api key's own slug + * + * The common case is a logged-in human: `me mcp` just works against the active + * space. Agents pass ME_API_KEY (the key embeds its space, so --space is + * optional). * * MCP registration with individual AI tools lives in per-agent commands: * me opencode install, me gemini install, me codex install * Claude Code uses the Memory Engine plugin instead of a CLI installer. */ import { Command } from "commander"; +import { resolveCredentials } from "../credentials.ts"; import { runMcpServer } from "../mcp/server.ts"; -const DEFAULT_SERVER = "https://api.memory.build"; +/** Extract the space slug embedded in an api key (`me...`). */ +function slugFromApiKey(token: string): string | undefined { + if (!token.startsWith("me.")) return undefined; + const parts = token.split("."); + return parts.length >= 4 ? parts[1] : undefined; +} -/** - * me mcp — run the MCP server over stdio. - * - * Does NOT use the credentials file. API key must be provided - * via --api-key or ME_API_KEY env var. - */ function createMcpRunAction() { return async (_opts: Record, cmd: Command) => { const opts = cmd.optsWithGlobals(); + const creds = resolveCredentials(opts.server as string | undefined); - // Resolve API key: --api-key > ME_API_KEY - const apiKey = - (opts.apiKey as string | undefined) ?? process.env.ME_API_KEY; - if (!apiKey) { + // Token: --api-key > ME_API_KEY (creds.apiKey) > stored session token. + const token = + (opts.apiKey as string | undefined) ?? creds.apiKey ?? creds.sessionToken; + if (!token) { console.error( - "Error: API key required. Provide via --api-key or ME_API_KEY env var.", + "Error: no credentials. Run 'me login', or pass --api-key / set ME_API_KEY.", ); process.exit(1); } - // Resolve server: --server > ME_SERVER > default - const server = - (opts.server as string | undefined) ?? - process.env.ME_SERVER ?? - DEFAULT_SERVER; + // Space: --space > ME_SPACE / stored active space > the api key's own slug. + const space = + (opts.space as string | undefined) ?? + creds.activeSpace ?? + slugFromApiKey(token); + if (!space) { + console.error( + "Error: no active space. Run 'me space use ', or pass --space / set ME_SPACE.", + ); + process.exit(1); + } - await runMcpServer({ apiKey, server }); + await runMcpServer({ server: creds.server, token, space }); }; } -/** - * Create the MCP command. - */ export function createMcpCommand(): Command { return new Command("mcp") .description("run MCP server over stdio") - .option("--api-key ", "API key for engine authentication") + .option("--api-key ", "agent api key (else uses the stored session)") + .option( + "--space ", + "active space (else ME_SPACE / stored active space)", + ) .action(createMcpRunAction()); } diff --git a/packages/cli/mcp/agent-install.ts b/packages/cli/mcp/agent-install.ts index fc1e62d..7ac5dd7 100644 --- a/packages/cli/mcp/agent-install.ts +++ b/packages/cli/mcp/agent-install.ts @@ -34,7 +34,9 @@ export async function runAgentMcpInstall( process.exit(1); } - // Resolve credentials: flags > credentials file + // Resolve credentials: flags > env (ME_API_KEY) > server default. MCP configs + // bake in a long-lived agent api key (a human session would expire), so an + // api key is required here — mint one with `me apikey create `. let { apiKey, server } = opts; if (!apiKey || !server) { const creds = resolveCredentials(server); @@ -44,7 +46,7 @@ export async function runAgentMcpInstall( if (!apiKey) { clack.log.error( - "No API key available. Either pass --api-key or run 'me engine use' first.", + "No API key available. Pass --api-key or set ME_API_KEY — mint one with 'me apikey create '.", ); process.exit(1); } diff --git a/packages/cli/mcp/server.ts b/packages/cli/mcp/server.ts index 8ba7cc3..fffd19d 100644 --- a/packages/cli/mcp/server.ts +++ b/packages/cli/mcp/server.ts @@ -14,8 +14,8 @@ import { stringify as yamlStringify } from "yaml"; import { z } from "zod"; import { CLIENT_VERSION } from "../../../version"; import { batchCreateChunked } from "../chunk.ts"; -import type { EngineClient } from "../client.ts"; -import { createClient } from "../client.ts"; +import type { MemoryClient } from "../client.ts"; +import { createMemoryClient } from "../client.ts"; import { formatMemoryAsMarkdown } from "../commands/memory.ts"; import { detectFormatFromExtension, @@ -47,7 +47,7 @@ Integration guide: ${DOCS_BASE}/mcp-integration.md`; // Tool Registration // ============================================================================= -function registerTools(server: McpServer, client: EngineClient): void { +function registerTools(server: McpServer, client: MemoryClient): void { // me_memory_create server.registerTool( "me_memory_create", @@ -967,15 +967,23 @@ function setupShutdownHandlers(mcpServer: McpServer): void { // ============================================================================= export interface McpServerOptions { - apiKey: string; + /** Base server URL. */ server: string; + /** Bearer token — a session token (human) or an agent api key. */ + token: string; + /** Active space slug (sent as X-Me-Space). */ + space: string; } /** * Run MCP server over stdio. */ export async function runMcpServer(options: McpServerOptions): Promise { - const client = createClient({ url: options.server, apiKey: options.apiKey }); + const client = createMemoryClient({ + url: options.server, + token: options.token, + space: options.space, + }); const mcpServer = new McpServer( { From 1ab1280d6928695314612f01029c71c10a16c438 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 18:09:19 +0200 Subject: [PATCH 058/156] feat(cli): retire the engine client; remove legacy command groups (4E-CLI-7) Migrate the remaining data commands off the engine api-key client onto the memory client (session + active space), then delete the legacy RBAC/account command groups and the now-dead engine/accounts plumbing. Migrated to the memory client (.memory.* is identical, so handlers unchanged): - pack install/list, agent-session import (claude/codex/gemini/opencode), the claude capture hook, and the importers writer. - me serve: the local web UI proxy now targets /api/v1/memory/rpc with the session token + X-Me-Space (was the engine RPC + api key). - capture/serve derive the space from the api key's slug when not given; HookConfig gains `space` (CLAUDE_PLUGIN_OPTION_SPACE override). Removed: - commands engine/org/invitation/user/owner/role/grant (replaced by space/group/access/agent/apikey). - client.ts createClient/createAccountsClient + Engine/Accounts types. - util.ts requireEngine + resolveOrg/resolveOrgId/resolveUserId/ resolveIdentityId/resolveMember. - credentials.ts engine-model fields/helpers (active_engine, engines, storeApiKey, setActiveEngine, getEngineApiKey); api keys come from ME_API_KEY only. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/chunk.ts | 2 +- packages/cli/claude/capture.test.ts | 11 +- packages/cli/claude/capture.ts | 29 +- packages/cli/client.ts | 33 +- packages/cli/commands/engine.ts | 470 ------------------------- packages/cli/commands/grant.ts | 183 ---------- packages/cli/commands/import.ts | 12 +- packages/cli/commands/invitation.ts | 199 ----------- packages/cli/commands/org.ts | 334 ------------------ packages/cli/commands/owner.ts | 148 -------- packages/cli/commands/pack.ts | 39 +- packages/cli/commands/role.ts | 282 --------------- packages/cli/commands/serve.ts | 18 +- packages/cli/commands/user.ts | 220 ------------ packages/cli/credentials.ts | 87 +---- packages/cli/importers/index.ts | 10 +- packages/cli/index.ts | 24 +- packages/cli/serve/http-server.test.ts | 18 +- packages/cli/serve/http-server.ts | 34 +- packages/cli/util.test.ts | 73 ++-- packages/cli/util.ts | 265 +------------- 21 files changed, 168 insertions(+), 2323 deletions(-) delete mode 100644 packages/cli/commands/engine.ts delete mode 100644 packages/cli/commands/grant.ts delete mode 100644 packages/cli/commands/invitation.ts delete mode 100644 packages/cli/commands/org.ts delete mode 100644 packages/cli/commands/owner.ts delete mode 100644 packages/cli/commands/role.ts delete mode 100644 packages/cli/commands/user.ts diff --git a/packages/cli/chunk.ts b/packages/cli/chunk.ts index e05dc32..ac2a4c6 100644 --- a/packages/cli/chunk.ts +++ b/packages/cli/chunk.ts @@ -108,7 +108,7 @@ export function* chunkMemoriesForBatchCreate( /** * Minimal client shape `batchCreateChunked` needs. Structurally typed so - * callers can pass an `EngineClient` or a stub in tests without coupling + * callers can pass a `MemoryClient` or a stub in tests without coupling * this module to the full client surface. */ export interface BatchCreateClient { diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index f8558bb..c14d7df 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -2,7 +2,7 @@ * Unit tests for Claude Code hook capture logic. */ import { describe, expect, test } from "bun:test"; -import type { EngineClient } from "@memory.build/client"; +import type { MemoryClient } from "@memory.build/client"; import { buildMeta, captureHookEvent, @@ -24,6 +24,7 @@ const BASE_EVENT = { const CONFIG: HookConfig = { server: "https://api.example.com", apiKey: "me.eng123.aaa.bbb", + space: "eng123", treePrefix: "claude_code.sessions", }; @@ -150,9 +151,9 @@ describe("buildMeta", () => { // captureHookEvent // ============================================================================= -/** Build a mock EngineClient that records the last memory.create call. */ +/** Build a mock MemoryClient that records the last memory.create call. */ function mockClient(): { - client: EngineClient; + client: MemoryClient; calls: Array>; } { const calls: Array> = []; @@ -163,7 +164,7 @@ function mockClient(): { return { id: "01960000-0000-7000-8000-000000000000" }; }, }, - } as unknown as EngineClient; + } as unknown as MemoryClient; return { client, calls }; } @@ -253,6 +254,7 @@ describe("resolveHookConfigFromEnv", () => { }); expect(cfg).toEqual({ apiKey: "me.eng.aaa.bbb", + space: "eng", server: "https://api.example.com", treePrefix: "my.prefix", }); @@ -264,6 +266,7 @@ describe("resolveHookConfigFromEnv", () => { }); expect(cfg).toEqual({ apiKey: "me.eng.aaa.bbb", + space: "eng", server: "https://api.memory.build", treePrefix: "claude_code.sessions", }); diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index bf6f179..3daa892 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -3,11 +3,11 @@ * * Pure functions for event parsing, project derivation, and metadata * construction are testable in isolation. The `captureHookEvent` entry - * point handles memory creation via EngineClient. + * point handles memory creation via the memory client. */ import { CLIENT_VERSION } from "../../../version"; -import { createClient, type EngineClient } from "../client.ts"; +import { createMemoryClient, type MemoryClient } from "../client.ts"; // ============================================================================= // Hook config (derived at runtime from CLAUDE_PLUGIN_OPTION_* env vars) @@ -16,12 +16,21 @@ import { createClient, type EngineClient } from "../client.ts"; export interface HookConfig { /** Memory Engine server URL. */ server: string; - /** API key (from the plugin's sensitive userConfig). */ + /** Agent api key (from the plugin's sensitive userConfig). */ apiKey: string; + /** Active space slug (X-Me-Space); defaults to the api key's own slug. */ + space: string; /** Tree path prefix for captured memories (ltree). */ treePrefix: string; } +/** Extract the space slug embedded in an api key (`me...`). */ +export function slugFromApiKey(apiKey: string): string | undefined { + if (!apiKey.startsWith("me.")) return undefined; + const parts = apiKey.split("."); + return parts.length >= 4 ? parts[1] : undefined; +} + // ============================================================================= // Event types // ============================================================================= @@ -166,8 +175,13 @@ export function resolveHookConfigFromEnv( const apiKey = env.CLAUDE_PLUGIN_OPTION_API_KEY; if (!apiKey) return null; + // The space defaults to the api key's own slug; an explicit env var overrides. + const space = env.CLAUDE_PLUGIN_OPTION_SPACE || slugFromApiKey(apiKey); + if (!space) return null; + return { apiKey, + space, server: env.CLAUDE_PLUGIN_OPTION_SERVER || DEFAULT_SERVER, treePrefix: env.CLAUDE_PLUGIN_OPTION_TREE_PREFIX || DEFAULT_TREE_PREFIX, }; @@ -185,7 +199,7 @@ export interface CaptureResult { export interface CaptureOptions { /** Override the client (for tests). */ - client?: EngineClient; + client?: MemoryClient; /** Override timestamp (for deterministic tests). */ now?: () => Date; } @@ -212,7 +226,12 @@ export async function captureHookEvent( const now = (opts.now ?? (() => new Date()))(); const client = - opts.client ?? createClient({ url: config.server, apiKey: config.apiKey }); + opts.client ?? + createMemoryClient({ + url: config.server, + token: config.apiKey, + space: config.space, + }); const result = await client.memory.create({ content, diff --git a/packages/cli/client.ts b/packages/cli/client.ts index 68d5401..6113e65 100644 --- a/packages/cli/client.ts +++ b/packages/cli/client.ts @@ -7,17 +7,11 @@ * import everything from one place. */ import { - type AccountsClient, - type AccountsClientOptions, type AuthClient, type AuthClientOptions, - createAccountsClient as baseCreateAccountsClient, createAuthClient as baseCreateAuthClient, - createClient as baseCreateClient, createMemoryClient as baseCreateMemoryClient, createUserClient as baseCreateUserClient, - type ClientOptions, - type EngineClient, type MemoryClient, type MemoryClientOptions, type UserClient, @@ -25,25 +19,6 @@ import { } from "@memory.build/client"; import { CLIENT_VERSION } from "../../version"; -/** - * Engine client factory with `clientVersion: CLIENT_VERSION` injected. - */ -export function createClient(options: ClientOptions = {}): EngineClient { - return baseCreateClient({ clientVersion: CLIENT_VERSION, ...options }); -} - -/** - * Accounts client factory with `clientVersion: CLIENT_VERSION` injected. - */ -export function createAccountsClient( - options: AccountsClientOptions = {}, -): AccountsClient { - return baseCreateAccountsClient({ - clientVersion: CLIENT_VERSION, - ...options, - }); -} - /** * Auth client factory. * @@ -56,7 +31,7 @@ export function createAuthClient(options: AuthClientOptions = {}): AuthClient { } /** - * Memory client factory (new model: space data-plane + management) with + * Memory client factory (space data-plane + management) with * `clientVersion: CLIENT_VERSION` injected. Talks to /api/v1/memory/rpc with the * active space carried as X-Me-Space. */ @@ -67,7 +42,7 @@ export function createMemoryClient( } /** - * User client factory (new model: agent lifecycle + space discovery) with + * User client factory (agent lifecycle + space discovery + whoami) with * `clientVersion: CLIENT_VERSION` injected. Talks to /api/v1/user/rpc. */ export function createUserClient(options: UserClientOptions = {}): UserClient { @@ -77,15 +52,11 @@ export function createUserClient(options: UserClientOptions = {}): UserClient { // Re-export types and helpers used across the CLI. Pass-through so command // files don't need to dual-import from "@memory.build/client". export { - type AccountsClient, - type AccountsClientOptions, type AuthClient, type AuthClientOptions, type CheckServerVersionOptions, - type ClientOptions, checkServerVersion, DeviceFlowError, - type EngineClient, isRpcError, type MemoryClient, type MemoryClientOptions, diff --git a/packages/cli/commands/engine.ts b/packages/cli/commands/engine.ts deleted file mode 100644 index de26e61..0000000 --- a/packages/cli/commands/engine.ts +++ /dev/null @@ -1,470 +0,0 @@ -/** - * me engine — engine management commands. - * - * - me engine list: List engines across all your orgs - * - me engine use [id-or-name]: Select the active engine - * - me engine create : Create a new engine in an org - * - me engine rename : Rename an engine - * - me engine delete : Permanently delete an engine - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; -import { - getEngineApiKey, - resolveCredentials, - setActiveEngine, - storeApiKey, -} from "../credentials.ts"; -import { - getOutputFormat, - type OutputFormat, - output, - table, -} from "../output.ts"; -import { handleError, requireSession, resolveOrgId } from "../util.ts"; - -// UUIDv7 pattern for argument detection -const UUIDV7_RE = - /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -/** - * Flattened engine info with org context. - */ -interface EngineInfo { - id: string; - slug: string; - name: string; - status: string; - orgId: string; - orgName: string; -} - -/** - * Fetch all engines across all the user's orgs. - */ -async function fetchAllEngines( - accounts: ReturnType, -): Promise { - const { orgs } = await accounts.org.list(); - const engines: EngineInfo[] = []; - - for (const org of orgs) { - const { engines: orgEngines } = await accounts.engine.list({ - orgId: org.id, - }); - for (const engine of orgEngines) { - engines.push({ - id: engine.id, - slug: engine.slug, - name: engine.name, - status: engine.status, - orgId: org.id, - orgName: org.name, - }); - } - } - - return engines; -} - -/** - * me engine list — list engines across all orgs. - */ -function createEngineListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list engines across all your organizations") - .action(async (_opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const engines = await fetchAllEngines(accounts); - - const data = { - engines: engines.map((e) => ({ - id: e.id, - slug: e.slug, - name: e.name, - status: e.status, - orgName: e.orgName, - active: e.slug === creds.activeEngine, - })), - }; - - output(data, fmt, () => { - if (engines.length === 0) { - console.log(" No engines found."); - return; - } - table( - ["id", "name", "slug", "org", "status"], - engines.map((e) => [ - e.id, - e.name, - e.slug, - e.orgName, - e.slug === creds.activeEngine ? `${e.status} (active)` : e.status, - ]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * Resolve an engine argument (ID, name, or interactive picker). - */ -async function resolveEngine( - engines: EngineInfo[], - arg: string | undefined, - fmt: OutputFormat, -): Promise { - if (!arg) { - // No argument — interactive picker - if (fmt !== "text") { - output( - { error: "Engine ID or name is required in non-interactive mode" }, - fmt, - () => {}, - ); - process.exit(1); - } - - if (engines.length === 0) { - clack.log.error("No engines found."); - process.exit(1); - } - - const selected = await clack.select({ - message: "Select an engine", - options: engines.map((e) => ({ - value: e.id, - label: `${e.name} — ${e.orgName}`, - hint: e.slug, - })), - }); - - if (clack.isCancel(selected)) { - clack.cancel("Cancelled."); - process.exit(0); - } - - return engines.find((e) => e.id === (selected as string)) ?? null; - } - - // Argument provided — try to match - if (UUIDV7_RE.test(arg)) { - // Looks like a UUID — match by ID - const match = engines.find((e) => e.id === arg); - if (!match) { - const msg = `No engine found with ID: ${arg}`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - return match; - } - - // Match by name - const matches = engines.filter((e) => e.name === arg); - if (matches.length === 0) { - const msg = `No engine named '${arg}' found`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - if (matches.length > 1) { - const msg = `Multiple engines named '${arg}'. Use the engine ID instead:`; - if (fmt === "text") { - clack.log.error(msg); - for (const m of matches) { - console.log(` ${m.id} — ${m.orgName}`); - } - } else { - output( - { - error: msg, - matches: matches.map((m) => ({ - id: m.id, - orgName: m.orgName, - slug: m.slug, - })), - }, - fmt, - () => {}, - ); - } - process.exit(1); - } - - return matches[0] ?? null; -} - -/** - * me engine use — select the active engine. - */ -function createEngineUseCommand(): Command { - return new Command("use") - .description("select the active engine") - .argument("[id-or-name]", "engine ID or name") - .action(async (arg: string | undefined, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const engines = await fetchAllEngines(accounts); - const engine = await resolveEngine(engines, arg, fmt); - if (!engine) { - process.exit(1); - } - - // Check if we already have an API key for this engine - const existingKey = getEngineApiKey(creds.server, engine.slug); - if (existingKey) { - // Already have a key — just switch active engine - setActiveEngine(creds.server, engine.slug); - output( - { - engine: engine.name, - slug: engine.slug, - org: engine.orgName, - switched: true, - }, - fmt, - () => { - clack.log.success( - `Switched to engine '${engine.name}' (${engine.orgName})`, - ); - }, - ); - return; - } - - // No key — call setupAccess - const spin = fmt === "text" ? clack.spinner() : null; - spin?.start("Setting up engine access..."); - - const result = await accounts.engine.setupAccess({ - engineId: engine.id, - }); - - // Store the API key and set active engine - storeApiKey(creds.server, result.engineSlug, result.rawKey); - - spin?.stop("Engine access configured."); - - output( - { - engine: result.engineName, - slug: result.engineSlug, - org: result.orgName, - userId: result.userId, - setup: true, - }, - fmt, - () => { - clack.log.success( - `Connected to engine '${result.engineName}' (${result.orgName})`, - ); - }, - ); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * me engine create — create a new engine in an org. - */ -function createEngineCreateCommand(): Command { - return new Command("create") - .description("create a new engine in an organization") - .argument("", "engine name") - .option("--org ", "organization ID") - .option("--language ", "text search language", "english") - .action(async (name: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const engine = await accounts.engine.create({ - orgId, - name, - language: opts.language, - }); - - output(engine, fmt, () => { - clack.log.success(`Created engine '${engine.name}'`); - console.log(` ID: ${engine.id}`); - console.log(` Slug: ${engine.slug}`); - console.log(` Status: ${engine.status}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * me engine rename — rename an engine. - * - * Renaming changes only the human-readable name. The engine slug - * (which backs the underlying `me_` schema and any stored API - * keys) remains unchanged, so the active engine selection and - * API-key access are unaffected. - */ -function createEngineRenameCommand(): Command { - return new Command("rename") - .description("rename an engine") - .argument("", "engine ID or name") - .argument("", "new engine name") - .action(async (idOrName: string, newName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const engines = await fetchAllEngines(accounts); - const engine = await resolveEngine(engines, idOrName, fmt); - if (!engine) { - handleError(new Error(`Engine not found: ${idOrName}`), fmt); - } - - const oldName = engine.name; - const updated = await accounts.engine.update({ - id: engine.id, - name: newName, - }); - - output(updated, fmt, () => { - clack.log.success( - `Renamed engine '${oldName}' → '${updated.name}' (${engine.orgName})`, - ); - console.log(` ID: ${updated.id}`); - console.log(` Slug: ${updated.slug}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * me engine delete — permanently delete an engine and all its data. - */ -function createEngineDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("permanently delete an engine and all its data") - .argument("", "engine ID or name") - .option("--force", "skip confirmation prompt") - .action(async (idOrName: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - // Resolve engine by ID or name - const engines = await fetchAllEngines(accounts); - const engine = await resolveEngine(engines, idOrName, fmt); - if (!engine) { - handleError(new Error(`Engine not found: ${idOrName}`), fmt); - } - - // Confirmation: require typing the engine name - if (fmt === "text" && !opts.force) { - clack.log.warn( - "This will permanently delete the engine and ALL its data (memories, users, grants).", - ); - clack.log.warn("This action cannot be undone."); - console.log(); - - const confirmation = await clack.text({ - message: `Type the engine name "${engine.name}" to confirm deletion`, - validate: (value) => { - if (value !== engine.name) { - return `Please type "${engine.name}" exactly to confirm`; - } - }, - }); - - if (clack.isCancel(confirmation)) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const result = await accounts.engine.delete({ id: engine.id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success( - `Engine '${engine.name}' has been permanently deleted.`, - ); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * Create the engine command group. - */ -export function createEngineCommand(): Command { - const engine = new Command("engine").description("manage engines"); - - engine.addCommand(createEngineListCommand()); - engine.addCommand(createEngineUseCommand()); - engine.addCommand(createEngineCreateCommand()); - engine.addCommand(createEngineRenameCommand()); - engine.addCommand(createEngineDeleteCommand()); - - return engine; -} diff --git a/packages/cli/commands/grant.ts b/packages/cli/commands/grant.ts deleted file mode 100644 index 1380bee..0000000 --- a/packages/cli/commands/grant.ts +++ /dev/null @@ -1,183 +0,0 @@ -/** - * me grant — tree grant management commands. - * - * - me grant create : Grant tree access - * - me grant revoke : Revoke tree access - * - me grant list [user]: List grants - * - me grant check : Check access - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createGrantCreateCommand(): Command { - return new Command("create") - .description("grant tree access to a user") - .argument("", "user name or ID") - .argument("", "tree path") - .argument("", "actions: read, create, update, delete") - .option("--with-grant-option", "allow grantee to re-grant") - .action( - async (user: string, path: string, actions: string[], opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.grant.create({ - userId, - treePath: path, - actions: actions as ("read" | "create" | "update" | "delete")[], - withGrantOption: opts.withGrantOption ?? false, - }); - - output(result, fmt, () => { - clack.log.success( - `Granted [${actions.join(", ")}] on '${path}' to ${user}`, - ); - }); - } catch (error) { - handleError(error, fmt); - } - }, - ); -} - -function createGrantRevokeCommand(): Command { - return new Command("revoke") - .description("revoke tree access from a user") - .argument("", "user name or ID") - .argument("", "tree path") - .action(async (user: string, path: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.grant.revoke({ - userId, - treePath: path, - }); - - output(result, fmt, () => { - if (result.revoked) { - clack.log.success(`Revoked grant on '${path}' from ${user}`); - } else { - clack.log.warn("Grant not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createGrantListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list grants") - .argument("[user]", "filter by user name or ID (optional)") - .action(async (user: string | undefined, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = user ? await resolveUserId(engine, user) : undefined; - const { grants } = await engine.grant.list( - userId ? { userId } : undefined, - ); - - output({ grants }, fmt, () => { - if (grants.length === 0) { - console.log(" No grants found."); - return; - } - table( - ["user", "tree_path", "actions", "grant_option"], - grants.map((g) => [ - g.userName, - g.treePath, - g.actions.join(", "), - g.withGrantOption ? "yes" : "", - ]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createGrantCheckCommand(): Command { - return new Command("check") - .description("check if a user has access to a tree path") - .argument("", "user name or ID") - .argument("", "tree path") - .argument("", "action: read, create, update, delete") - .action(async (user: string, path: string, action: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.grant.check({ - userId, - treePath: path, - action: action as "read" | "create" | "update" | "delete", - }); - - output(result, fmt, () => { - if (result.allowed) { - clack.log.success(`${action} on '${path}': allowed`); - } else { - clack.log.warn(`${action} on '${path}': denied`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createGrantCommand(): Command { - const grant = new Command("grant").description("manage tree grants"); - grant.addCommand(createGrantCreateCommand()); - grant.addCommand(createGrantRevokeCommand()); - grant.addCommand(createGrantListCommand()); - grant.addCommand(createGrantCheckCommand()); - return grant; -} diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index 3a788b7..b04ee9d 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -24,7 +24,6 @@ */ import * as clack from "@clack/prompts"; import { Command } from "commander"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { createProgressReporter, @@ -35,7 +34,12 @@ import { } from "../importers/index.ts"; import type { ImporterOptions } from "../importers/types.ts"; import { getOutputFormat, output } from "../output.ts"; -import { handleError, requireEngine, requireSession } from "../util.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, +} from "../util.ts"; const DEFAULT_TREE_ROOT = "projects"; const DEFAULT_SESSIONS_NODE_NAME = "agent_sessions"; @@ -161,7 +165,7 @@ async function runAndRender( ); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); let config: ReturnType; try { @@ -170,7 +174,7 @@ async function runAndRender( handleError(error, fmt); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const engine = buildMemoryClient(creds); if (fmt === "text" && config.write.verbose) { const sourceNote = config.importer.source ?? importer.defaultSource; diff --git a/packages/cli/commands/invitation.ts b/packages/cli/commands/invitation.ts deleted file mode 100644 index 68a69aa..0000000 --- a/packages/cli/commands/invitation.ts +++ /dev/null @@ -1,199 +0,0 @@ -/** - * me invitation — invitation management commands. - * - * - me invitation create : Invite someone to an organization - * - me invitation list [org-id]: List pending invitations - * - me invitation accept : Accept an invitation - * - me invitation revoke : Revoke an invitation - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { handleError, requireSession, resolveOrgId } from "../util.ts"; - -// ============================================================================= -// Invitation Commands -// ============================================================================= - -function createInvitationCreateCommand(): Command { - return new Command("create") - .description("invite someone to an organization") - .argument("", "email address to invite") - .argument("", "role: owner, admin, or member") - .option("--org ", "organization name, slug, or ID") - .option("--expires ", "expiration in days (1-30, default 7)", "7") - .action(async (email: string, role: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const expiresInDays = Number.parseInt(opts.expires, 10); - - const result = await accounts.invitation.create({ - orgId, - email, - role: role as "owner" | "admin" | "member", - expiresInDays, - }); - - output(result, fmt, () => { - clack.log.success(`Invitation sent to ${result.email}`); - console.log(` ID: ${result.id}`); - console.log(` Role: ${result.role}`); - console.log(` Expires: ${result.expiresAt}`); - clack.note( - result.token, - "Invitation token (share this with the invitee)", - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createInvitationListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list pending invitations") - .argument("[org]", "organization name, slug, or ID") - .option("--org ", "organization name, slug, or ID") - .action(async (positionalOrgId: string | undefined, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId( - accounts, - fmt, - opts.org, - positionalOrgId, - ); - const { invitations } = await accounts.invitation.list({ orgId }); - - output({ invitations }, fmt, () => { - if (invitations.length === 0) { - console.log(" No pending invitations."); - return; - } - table( - ["id", "email", "role", "expires"], - invitations.map((inv) => [ - inv.id, - inv.email, - inv.role, - inv.expiresAt, - ]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createInvitationAcceptCommand(): Command { - return new Command("accept") - .description("accept an invitation") - .argument("", "invitation token") - .action(async (token: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const result = await accounts.invitation.accept({ token }); - - // Resolve org name for a friendlier message - let orgName: string | undefined; - if (result.accepted) { - try { - const org = await accounts.org.get({ id: result.orgId }); - orgName = org.name; - } catch { - // Fall back to ID if org lookup fails - } - } - - output(result, fmt, () => { - if (result.accepted) { - const label = orgName ? `'${orgName}'` : result.orgId; - clack.log.success(`Invitation accepted! Joined ${label}.`); - } else { - clack.log.warn("Invitation could not be accepted."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createInvitationRevokeCommand(): Command { - return new Command("revoke") - .description("revoke a pending invitation") - .argument("", "invitation ID") - .action(async (id: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const result = await accounts.invitation.revoke({ id }); - - output(result, fmt, () => { - if (result.revoked) { - clack.log.success("Invitation revoked."); - } else { - clack.log.warn("Invitation not found or already used."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -// ============================================================================= -// Command Group -// ============================================================================= - -export function createInvitationCommand(): Command { - const invitation = new Command("invitation").description( - "manage invitations", - ); - invitation.addCommand(createInvitationCreateCommand()); - invitation.addCommand(createInvitationListCommand()); - invitation.addCommand(createInvitationAcceptCommand()); - invitation.addCommand(createInvitationRevokeCommand()); - return invitation; -} diff --git a/packages/cli/commands/org.ts b/packages/cli/commands/org.ts deleted file mode 100644 index 2cae998..0000000 --- a/packages/cli/commands/org.ts +++ /dev/null @@ -1,334 +0,0 @@ -/** - * me org — organization management commands. - * - * - me org list: List your organizations - * - me org create : Create an organization - * - me org rename : Rename an organization - * - me org delete : Delete an organization - * - me org member list [org]: List members - * - me org member add : Add a member - * - me org member remove : Remove a member - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireSession, - resolveIdentityId, - resolveMember, - resolveOrg, - resolveOrgId, -} from "../util.ts"; - -// ============================================================================= -// Org Commands -// ============================================================================= - -function createOrgListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list your organizations") - .action(async (_opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const { orgs } = await accounts.org.list(); - - output({ orgs }, fmt, () => { - if (orgs.length === 0) { - console.log(" No organizations found."); - return; - } - table( - ["id", "name", "slug"], - orgs.map((org) => [org.id, org.name, org.slug]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgCreateCommand(): Command { - return new Command("create") - .description("create an organization") - .argument("", "organization name") - .action(async (name: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const org = await accounts.org.create({ name }); - - output(org, fmt, () => { - clack.log.success(`Created organization '${org.name}'`); - console.log(` ID: ${org.id}`); - console.log(` Slug: ${org.slug}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgRenameCommand(): Command { - return new Command("rename") - .description("rename an organization") - .argument("", "organization name, slug, or ID") - .argument("", "new organization name") - .action(async (nameOrId: string, newName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const org = await resolveOrg(accounts, fmt, undefined, nameOrId); - const oldName = org.name; - const updated = await accounts.org.update({ - id: org.id, - name: newName, - }); - - output(updated, fmt, () => { - clack.log.success( - `Renamed organization '${oldName}' → '${updated.name}'`, - ); - console.log(` ID: ${updated.id}`); - console.log(` Slug: ${updated.slug}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("delete an organization") - .argument("", "organization name, slug, or ID") - .option("-y, --yes", "skip confirmation prompt") - .action(async (nameOrId: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const org = await resolveOrg(accounts, fmt, undefined, nameOrId); - - // Confirm in text mode unless --yes - if (fmt === "text" && !opts.yes) { - const confirmed = await clack.confirm({ - message: `Delete organization '${org.name}'? This cannot be undone.`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const result = await accounts.org.delete({ id: org.id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success(`Organization '${org.name}' deleted.`); - } else { - clack.log.warn("Organization not found."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -// ============================================================================= -// Org Member Commands -// ============================================================================= - -function createOrgMemberListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list organization members") - .argument("[org]", "organization name, slug, or ID") - .option("--org ", "organization name, slug, or ID") - .action(async (positionalOrgId: string | undefined, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId( - accounts, - fmt, - opts.org, - positionalOrgId, - ); - const { members } = await accounts.org.member.list({ orgId }); - - output({ members }, fmt, () => { - if (members.length === 0) { - console.log(" No members found."); - return; - } - table( - ["name", "email", "role", "joined"], - members.map((m) => [m.name, m.email, m.role, m.createdAt]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgMemberAddCommand(): Command { - return new Command("add") - .description("add a member to an organization") - .argument("", "email address or identity ID") - .argument("", "role: owner, admin, or member") - .option("--org ", "organization name, slug, or ID") - .action(async (emailOrId: string, role: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const identityId = await resolveIdentityId(accounts, fmt, emailOrId); - const member = await accounts.org.member.add({ - orgId, - identityId, - role: role as "owner" | "admin" | "member", - }); - - output(member, fmt, () => { - clack.log.success( - `Added ${member.name} (${member.email}) as ${member.role}`, - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgMemberRemoveCommand(): Command { - return new Command("remove") - .description("remove a member from an organization") - .argument("", "member name, email, or identity ID") - .option("--org ", "organization name, slug, or ID") - .option("-y, --yes", "skip confirmation prompt") - .action(async (nameEmailOrId: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const member = await resolveMember(accounts, fmt, orgId, nameEmailOrId); - - // Confirm in text mode unless --yes - if (fmt === "text" && !opts.yes) { - const label = member.email - ? `${member.name} (${member.email})` - : member.name; - const confirmed = await clack.confirm({ - message: `Remove ${label}?`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const result = await accounts.org.member.remove({ - orgId, - identityId: member.identityId, - }); - - output(result, fmt, () => { - if (result.removed) { - clack.log.success(`Removed ${member.name}.`); - } else { - clack.log.warn("Member not found."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -// ============================================================================= -// Command Group -// ============================================================================= - -function createOrgMemberCommand(): Command { - const member = new Command("member").description( - "manage organization members", - ); - member.addCommand(createOrgMemberListCommand()); - member.addCommand(createOrgMemberAddCommand()); - member.addCommand(createOrgMemberRemoveCommand()); - return member; -} - -export function createOrgCommand(): Command { - const org = new Command("org").description("manage organizations"); - org.addCommand(createOrgListCommand()); - org.addCommand(createOrgCreateCommand()); - org.addCommand(createOrgRenameCommand()); - org.addCommand(createOrgDeleteCommand()); - org.addCommand(createOrgMemberCommand()); - return org; -} diff --git a/packages/cli/commands/owner.ts b/packages/cli/commands/owner.ts deleted file mode 100644 index ba9773c..0000000 --- a/packages/cli/commands/owner.ts +++ /dev/null @@ -1,148 +0,0 @@ -/** - * me owner — tree ownership management commands. - * - * - me owner set : Set tree path owner - * - me owner remove : Remove tree path owner - * - me owner get : Get tree path owner - * - me owner list [user]: List ownership records - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createOwnerSetCommand(): Command { - return new Command("set") - .description("set tree path owner") - .argument("", "tree path") - .argument("", "user name or ID") - .action(async (path: string, user: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.owner.set({ userId, treePath: path }); - - output(result, fmt, () => { - clack.log.success(`Set owner of '${path}' to ${user}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createOwnerRemoveCommand(): Command { - return new Command("remove") - .description("remove tree path owner") - .argument("", "tree path") - .action(async (path: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const result = await engine.owner.remove({ treePath: path }); - - output(result, fmt, () => { - if (result.removed) { - clack.log.success(`Removed owner of '${path}'`); - } else { - clack.log.warn(`No owner found for '${path}'`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createOwnerGetCommand(): Command { - return new Command("get") - .description("get tree path owner") - .argument("", "tree path") - .action(async (path: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const owner = await engine.owner.get({ treePath: path }); - - output(owner, fmt, () => { - console.log(` Path: ${owner.treePath}`); - console.log(` Owner: ${owner.userName}`); - console.log(` Set by: ${owner.createdByName ?? "(unknown)"}`); - console.log(` Created: ${owner.createdAt}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createOwnerListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list ownership records") - .argument("[user]", "filter by user name or ID (optional)") - .action(async (user: string | undefined, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = user ? await resolveUserId(engine, user) : undefined; - const { owners } = await engine.owner.list( - userId ? { userId } : undefined, - ); - - output({ owners }, fmt, () => { - if (owners.length === 0) { - console.log(" No ownership records found."); - return; - } - table( - ["tree_path", "owner"], - owners.map((o) => [o.treePath, o.userName]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createOwnerCommand(): Command { - const owner = new Command("owner").description("manage tree ownership"); - owner.addCommand(createOwnerSetCommand()); - owner.addCommand(createOwnerRemoveCommand()); - owner.addCommand(createOwnerGetCommand()); - owner.addCommand(createOwnerListCommand()); - return owner; -} diff --git a/packages/cli/commands/pack.ts b/packages/cli/commands/pack.ts index 0deddfb..7f4296e 100644 --- a/packages/cli/commands/pack.ts +++ b/packages/cli/commands/pack.ts @@ -2,19 +2,23 @@ * me pack — memory pack management commands. * * - me pack validate : Validate a pack file locally - * - me pack install : Install a memory pack into the active engine - * - me pack list: List installed packs in the active engine + * - me pack install : Install a memory pack into the active space + * - me pack list: List installed packs in the active space */ import { readFileSync } from "node:fs"; import * as clack from "@clack/prompts"; import { Command } from "commander"; import { batchCreateChunked } from "../chunk.ts"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; import { parsePack, validatePackConstraints } from "../parsers/pack.ts"; -import { handleError, requireEngine, requireSession } from "../util.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, +} from "../util.ts"; // ============================================================================= // Validate @@ -91,7 +95,7 @@ function createPackValidateCommand(): Command { function createPackInstallCommand(): Command { return new Command("install") - .description("install a memory pack into the active engine") + .description("install a memory pack into the active space") .argument("", "pack YAML file to install") .option("--dry-run", "preview what would happen without making changes") .option("-y, --yes", "skip confirmation for stale memory deletion") @@ -100,7 +104,7 @@ function createPackInstallCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); try { // Step 1: Read and validate @@ -126,14 +130,11 @@ function createPackInstallCommand(): Command { const packName = envelope.name; const packVersion = envelope.version; - // Step 2: Connect to engine - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); + // Step 2: connect to the active space + const client = buildMemoryClient(creds); // Step 3: Search for existing memories with same pack name - const existing = await engine.memory.search({ + const existing = await client.memory.search({ meta: { pack: { name: packName } }, limit: 1000, }); @@ -215,7 +216,7 @@ function createPackInstallCommand(): Command { ); for (const mem of stale) { - await engine.memory.delete({ id: mem.id }); + await client.memory.delete({ id: mem.id }); } spin?.stop( @@ -247,7 +248,7 @@ function createPackInstallCommand(): Command { insertedIds, failedIds, errors: chunkErrors, - } = await batchCreateChunked(engine, createParams); + } = await batchCreateChunked(client, createParams); spin?.stop("Done"); @@ -360,19 +361,19 @@ function createPackInstallCommand(): Command { function createPackListCommand(): Command { return new Command("list") .alias("ls") - .description("list installed packs in the active engine") + .description("list installed packs in the active space") .action(async (_opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { // Search for all memories with meta.pack - const result = await engine.memory.search({ + const result = await client.memory.search({ meta: { pack: {} }, limit: 1000, }); @@ -424,7 +425,7 @@ function createPackListCommand(): Command { // ============================================================================= /** - * `engine.memory.batchCreate` uses `ON CONFLICT (id) DO NOTHING` server-side, + * `client.memory.batchCreate` uses `ON CONFLICT (id) DO NOTHING` server-side, * so the returned `ids` array can be shorter than the request when conflicts * occur. For pack install, ids that didn't land fall into three buckets: * diff --git a/packages/cli/commands/role.ts b/packages/cli/commands/role.ts deleted file mode 100644 index 17e3654..0000000 --- a/packages/cli/commands/role.ts +++ /dev/null @@ -1,282 +0,0 @@ -/** - * me role — role management commands. - * - * - me role create : Create a role - * - me role delete : Delete a role (alias: rm) - * - me role list: List all roles - * - me role add-member : Add user to role (by ID or name) - * - me role remove-member : Remove user from role (by ID or name) - * - me role members : List role members (by ID or name) - * - me role list-for : List roles a user belongs to (by ID or name) - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createRoleCreateCommand(): Command { - return new Command("create") - .description("create a role") - .argument("", "role name") - .option("--identity-id ", "link to an accounts identity") - .action(async (name: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const role = await engine.role.create({ - name, - identityId: opts.identityId ?? undefined, - }); - - output(role, fmt, () => { - clack.log.success(`Created role '${role.name}'`); - console.log(` ID: ${role.id}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("delete a role") - .argument("", "role ID or name") - .option("-y, --yes", "skip confirmation prompt") - .action(async (idOrName: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - if (fmt === "text" && !opts.yes) { - const confirmed = await clack.confirm({ - message: `Delete role ${idOrName}? This removes all grants and memberships. This cannot be undone.`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const result = await engine.user.delete({ id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success("Role deleted."); - } else { - clack.log.warn("Role not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list all roles") - .action(async (_opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - // Roles are users with canLogin=false - const { users: roles } = await engine.user.list({ canLogin: false }); - - output({ roles }, fmt, () => { - if (roles.length === 0) { - console.log(" No roles found."); - return; - } - table( - ["id", "name"], - roles.map((r) => [r.id, r.name]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleAddMemberCommand(): Command { - return new Command("add-member") - .description("add a user to a role") - .argument("", "role ID or name") - .argument("", "member ID or name") - .option("--with-admin-option", "allow member to manage this role") - .action(async (role: string, member: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const [roleId, memberId] = await Promise.all([ - resolveUserId(engine, role), - resolveUserId(engine, member), - ]); - - const result = await engine.role.addMember({ - roleId, - memberId, - withAdminOption: opts.withAdminOption ?? false, - }); - - output(result, fmt, () => { - if (result.added) { - clack.log.success(`Added ${member} to role ${role}`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleRemoveMemberCommand(): Command { - return new Command("remove-member") - .description("remove a user from a role") - .argument("", "role ID or name") - .argument("", "member ID or name") - .action(async (role: string, member: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const [roleId, memberId] = await Promise.all([ - resolveUserId(engine, role), - resolveUserId(engine, member), - ]); - - const result = await engine.role.removeMember({ roleId, memberId }); - - output(result, fmt, () => { - if (result.removed) { - clack.log.success(`Removed ${member} from role ${role}`); - } else { - clack.log.warn("Membership not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleMembersCommand(): Command { - return new Command("members") - .description("list members of a role") - .argument("", "role ID or name") - .action(async (role: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const roleId = await resolveUserId(engine, role); - const { members } = await engine.role.listMembers({ roleId }); - - output({ members }, fmt, () => { - if (members.length === 0) { - console.log(" No members found."); - return; - } - table( - ["member_id", "name", "admin"], - members.map((m) => [ - m.memberId, - m.memberName, - m.withAdminOption ? "yes" : "", - ]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleListForCommand(): Command { - return new Command("list-for") - .description("list roles a user belongs to") - .argument("", "user ID or name") - .action(async (user: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const { roles } = await engine.role.listForUser({ userId }); - - output({ roles }, fmt, () => { - if (roles.length === 0) { - console.log(" No roles found."); - return; - } - table( - ["id", "name", "admin"], - roles.map((r) => [r.id, r.name, r.withAdminOption ? "yes" : ""]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createRoleCommand(): Command { - const role = new Command("role").description("manage roles"); - role.addCommand(createRoleCreateCommand()); - role.addCommand(createRoleDeleteCommand()); - role.addCommand(createRoleListCommand()); - role.addCommand(createRoleAddMemberCommand()); - role.addCommand(createRoleRemoveMemberCommand()); - role.addCommand(createRoleMembersCommand()); - role.addCommand(createRoleListForCommand()); - return role; -} diff --git a/packages/cli/commands/serve.ts b/packages/cli/commands/serve.ts index 9580c81..0573808 100644 --- a/packages/cli/commands/serve.ts +++ b/packages/cli/commands/serve.ts @@ -3,7 +3,8 @@ * * Launches a local HTTP server that: * - serves the embedded Vite-built React app - * - proxies POST /rpc to the configured engine, injecting the stored API key + * - proxies POST /rpc to the space memory endpoint, injecting the session token + * and the active space (X-Me-Space) * * Usage: * me serve [--port ] [--host ] [--no-open] @@ -16,7 +17,7 @@ import { Command } from "commander"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output } from "../output.ts"; import { findAvailablePort, startHttpServer } from "../serve/http-server.ts"; -import { requireEngine } from "../util.ts"; +import { requireSession, requireSpace } from "../util.ts"; const DEFAULT_PORT = 3000; const DEFAULT_HOST = "127.0.0.1"; @@ -40,7 +41,8 @@ export function createServeCommand(): Command { const fmt = getOutputFormat(globalOpts); const creds = resolveCredentials(globalOpts.server); - requireEngine(creds, fmt); + requireSession(creds, fmt); + requireSpace(creds, fmt); const host: string = opts.host ?? DEFAULT_HOST; const explicitPortFlag = opts.port !== undefined; @@ -74,8 +76,8 @@ export function createServeCommand(): Command { try { running = startHttpServer({ server: creds.server, - apiKey: creds.apiKey, - engineSlug: creds.activeEngine ?? "", + token: creds.sessionToken, + space: creds.activeSpace, host, port, }); @@ -95,9 +97,7 @@ export function createServeCommand(): Command { if (fmt === "text") { clack.log.success(`Memory Engine UI running at ${running.url}`); console.log(` Remote server: ${creds.server}`); - if (creds.activeEngine) { - console.log(` Active engine: ${creds.activeEngine}`); - } + console.log(` Active space: ${creds.activeSpace}`); console.log(" Press Ctrl+C to stop."); } else { output( @@ -106,7 +106,7 @@ export function createServeCommand(): Command { host, port: port, server: creds.server, - engine: creds.activeEngine, + space: creds.activeSpace, }, fmt, () => {}, diff --git a/packages/cli/commands/user.ts b/packages/cli/commands/user.ts deleted file mode 100644 index dca80e3..0000000 --- a/packages/cli/commands/user.ts +++ /dev/null @@ -1,220 +0,0 @@ -/** - * me user — engine user management commands. - * - * - me user list: List users in the active engine - * - me user create : Create an engine user - * - me user get : Get user by ID or name - * - me user delete : Delete a user (by ID or name) - * - me user rename : Rename a user (by ID or name) - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createUserListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list users in the active engine") - .option("--login-only", "only show users that can login") - .action(async (opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const { users } = await engine.user.list( - opts.loginOnly ? { canLogin: true } : undefined, - ); - - output({ users }, fmt, () => { - if (users.length === 0) { - console.log(" No users found."); - return; - } - table( - ["id", "name", "flags"], - users.map((u) => { - const flags = [ - u.superuser ? "superuser" : "", - u.createrole ? "createrole" : "", - !u.canLogin ? "role" : "", - ] - .filter(Boolean) - .join(", "); - return [u.id, u.name, flags]; - }), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserCreateCommand(): Command { - return new Command("create") - .description("create an engine user") - .argument("", "user name") - .option("--superuser", "grant superuser privileges") - .option("--createrole", "can create other users/roles") - .option("--no-login", "create as role (cannot authenticate)") - .option("--identity-id ", "link to an accounts identity") - .action(async (name: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const user = await engine.user.create({ - name, - superuser: opts.superuser ?? false, - createrole: opts.createrole ?? false, - canLogin: opts.login !== false, - identityId: opts.identityId ?? undefined, - }); - - output(user, fmt, () => { - clack.log.success(`Created user '${user.name}'`); - console.log(` ID: ${user.id}`); - console.log(` Superuser: ${user.superuser}`); - console.log(` Can Login: ${user.canLogin}`); - if (user.identityId) { - console.log(` Identity: ${user.identityId}`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserGetCommand(): Command { - return new Command("get") - .description("get a user by ID or name") - .argument("", "user ID (UUIDv7) or name") - .action(async (idOrName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const user = await engine.user.get({ id }); - - output(user, fmt, () => { - console.log(` Name: ${user.name}`); - console.log(` ID: ${user.id}`); - console.log(` Superuser: ${user.superuser}`); - console.log(` Createrole: ${user.createrole}`); - console.log(` Can Login: ${user.canLogin}`); - console.log(` Identity: ${user.identityId ?? "(none)"}`); - console.log(` Created: ${user.createdAt}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("delete a user") - .argument("", "user ID or name") - .option("-y, --yes", "skip confirmation prompt") - .action(async (idOrName: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - if (fmt === "text" && !opts.yes) { - const confirmed = await clack.confirm({ - message: `Delete user ${idOrName}? This cannot be undone.`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const result = await engine.user.delete({ id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success("User deleted."); - } else { - clack.log.warn("User not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserRenameCommand(): Command { - return new Command("rename") - .description("rename a user") - .argument("", "user ID or name") - .argument("", "new name") - .action(async (idOrName: string, newName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const result = await engine.user.rename({ id, name: newName }); - - output(result, fmt, () => { - if (result.renamed) { - clack.log.success(`User renamed to '${newName}'.`); - } else { - clack.log.warn("User not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createUserCommand(): Command { - const user = new Command("user").description("manage engine users"); - user.addCommand(createUserListCommand()); - user.addCommand(createUserCreateCommand()); - user.addCommand(createUserGetCommand()); - user.addCommand(createUserDeleteCommand()); - user.addCommand(createUserRenameCommand()); - return user; -} diff --git a/packages/cli/credentials.ts b/packages/cli/credentials.ts index 65973ac..835f743 100644 --- a/packages/cli/credentials.ts +++ b/packages/cli/credentials.ts @@ -22,9 +22,6 @@ * Linux `secret-tool`, Windows credential manager) with a fall back to this 0600 * file when no keychain is available (CI, headless Linux). The file would then * hold only non-secret pointers (default_server, active_space). - * - * The `active_engine` / `engines` fields are the legacy engine-model shape; they - * are read for backward compatibility but new logins write the space shape. */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; @@ -41,13 +38,6 @@ export const DEFAULT_SERVER = "https://api.memory.build"; // Types // ============================================================================= -/** - * Per-engine credential entry (legacy engine model). - */ -export interface EngineCredentials { - api_key: string; -} - /** * Per-server credential entry. */ @@ -55,9 +45,6 @@ export interface ServerCredentials { session_token?: string; /** Active space slug (sent as X-Me-Space). */ active_space?: string; - /** Legacy engine model — read for back-compat; new logins write `active_space`. */ - active_engine?: string; - engines?: Record; } /** @@ -74,11 +61,10 @@ export interface CredentialsFile { export interface ResolvedCredentials { server: string; sessionToken?: string; + /** Agent api key — ME_API_KEY only; never persisted. */ apiKey?: string; /** Active space slug (the X-Me-Space) — ME_SPACE env > stored active_space. */ activeSpace?: string; - /** Legacy engine model. */ - activeEngine?: string; } // ============================================================================= @@ -212,56 +198,6 @@ export function storeSessionToken(server: string, token: string): void { writeCredentials(creds); } -/** - * Store an API key for an engine on a server. - * Also sets the engine as active. - */ -export function storeApiKey( - server: string, - engineSlug: string, - apiKey: string, -): void { - const creds = readCredentials(); - const origin = normalizeOrigin(server); - - if (!creds.servers[origin]) { - creds.servers[origin] = {}; - } - if (!creds.servers[origin].engines) { - creds.servers[origin].engines = {}; - } - creds.servers[origin].engines[engineSlug] = { api_key: apiKey }; - creds.servers[origin].active_engine = engineSlug; - - writeCredentials(creds); -} - -/** - * Set the active engine for a server (without modifying API keys). - */ -export function setActiveEngine(server: string, engineSlug: string): void { - const creds = readCredentials(); - const origin = normalizeOrigin(server); - - if (!creds.servers[origin]) { - creds.servers[origin] = {}; - } - creds.servers[origin].active_engine = engineSlug; - - writeCredentials(creds); -} - -/** - * Get the API key for a specific engine on a server. - */ -export function getEngineApiKey( - server: string, - engineSlug: string, -): string | undefined { - const stored = getServerCredentials(server); - return stored.engines?.[engineSlug]?.api_key; -} - // ============================================================================= // Space Operations (new model) // ============================================================================= @@ -366,29 +302,18 @@ export function resolveServer(flagValue?: string): string { /** * Resolve all credentials for the active server. * - * For each credential type, env vars take priority over the stored file. - * API key is resolved from the active engine's stored key. + * The session token (ME_SESSION_TOKEN env > stored) authenticates humans; the + * active space (ME_SPACE env > stored active_space) is the X-Me-Space. An agent + * api key is never persisted — it only ever comes from ME_API_KEY. */ export function resolveCredentials(serverFlag?: string): ResolvedCredentials { const server = resolveServer(serverFlag); const stored = getServerCredentials(server); - // Resolve API key: env var > active engine's stored key - const activeEngine = stored.active_engine; - const storedApiKey = activeEngine - ? stored.engines?.[activeEngine]?.api_key - : undefined; - - // New model: the active space (X-Me-Space); ME_SPACE overrides the stored - // active_space. Api keys are never stored — an agent key only ever comes from - // ME_API_KEY (the legacy engine key remains as a fallback until removed). - const activeSpace = process.env.ME_SPACE ?? stored.active_space; - return { server, sessionToken: process.env.ME_SESSION_TOKEN ?? stored.session_token, - apiKey: process.env.ME_API_KEY ?? storedApiKey, - activeSpace, - activeEngine, + apiKey: process.env.ME_API_KEY, + activeSpace: process.env.ME_SPACE ?? stored.active_space, }; } diff --git a/packages/cli/importers/index.ts b/packages/cli/importers/index.ts index 521b190..8c50aec 100644 --- a/packages/cli/importers/index.ts +++ b/packages/cli/importers/index.ts @@ -16,7 +16,7 @@ import type { MemoryCreateParams } from "@memory.build/protocol/engine"; import { batchCreateChunked } from "../chunk.ts"; -import type { EngineClient } from "../client.ts"; +import type { MemoryClient } from "../client.ts"; import type { ProgressReporter } from "./progress.ts"; import { SlugRegistry } from "./slug.ts"; import { renderMessageContent, synthesizeTitle } from "./transcript.ts"; @@ -98,7 +98,7 @@ export interface WriteOptions { /** Run discovery + writes for a single importer. */ export async function runImport( - engine: EngineClient, + engine: MemoryClient, importer: Importer, importerOptions: ImporterOptions, writeOptions: WriteOptions, @@ -176,7 +176,7 @@ export async function runImport( * updates are issued one at a time (rare path). */ async function writeSession( - engine: EngineClient, + engine: MemoryClient, session: ImportedSession, title: string, tree: string, @@ -352,7 +352,7 @@ async function writeSession( * `undefined` when the record was written before the field existed). */ async function fetchExistingMessageVersions( - engine: EngineClient, + engine: MemoryClient, session: ImportedSession, tree: string, ): Promise> { @@ -448,7 +448,7 @@ function logOutcome( /** * Drop items whose `memoryId` has already been seen, preserving order. * Exported so the dedup behavior can be unit-tested without standing up - * a fake EngineClient. Used by `writeSession` to absorb sessions whose + * a fake MemoryClient. Used by `writeSession` to absorb sessions whose * JSONL has duplicate `event.uuid` entries (which would otherwise produce * two planned memories with the same deterministic UUIDv7). */ diff --git a/packages/cli/index.ts b/packages/cli/index.ts index 5fbfc3d..7db4219 100755 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -14,11 +14,8 @@ import { createAgentCommand } from "./commands/agent.ts"; import { createApiKeyCommand } from "./commands/apikey.ts"; import { createClaudeCommand } from "./commands/claude.ts"; import { createCodexCommand } from "./commands/codex.ts"; -import { createEngineCommand } from "./commands/engine.ts"; import { createGeminiCommand } from "./commands/gemini.ts"; -import { createGrantCommand } from "./commands/grant.ts"; import { createGroupCommand } from "./commands/group.ts"; -import { createInvitationCommand } from "./commands/invitation.ts"; import { createLoginCommand } from "./commands/login.ts"; import { createLogoutCommand } from "./commands/logout.ts"; import { createMcpCommand } from "./commands/mcp.ts"; @@ -27,14 +24,10 @@ import { createMemoryCommand, } from "./commands/memory.ts"; import { createOpenCodeCommand } from "./commands/opencode.ts"; -import { createOrgCommand } from "./commands/org.ts"; -import { createOwnerCommand } from "./commands/owner.ts"; import { createPackCommand } from "./commands/pack.ts"; -import { createRoleCommand } from "./commands/role.ts"; import { createServeCommand } from "./commands/serve.ts"; import { createSpaceCommand } from "./commands/space.ts"; import { createUpgradeCommand } from "./commands/upgrade.ts"; -import { createUserCommand } from "./commands/user.ts"; import { createVersionCommand } from "./commands/version.ts"; import { createWhoamiCommand } from "./commands/whoami.ts"; import { setExpanded } from "./output.ts"; @@ -71,22 +64,13 @@ program.addCommand(createWhoamiCommand()); program.addCommand(createVersionCommand()); program.addCommand(createUpgradeCommand()); -// Engine commands -program.addCommand(createEngineCommand()); - -// Space commands (new model) +// Space commands (the new model: spaces, groups, access, agents, api keys) program.addCommand(createSpaceCommand()); program.addCommand(createGroupCommand()); program.addCommand(createAccessCommand()); program.addCommand(createAgentCommand()); program.addCommand(createApiKeyCommand()); -// Org commands -program.addCommand(createOrgCommand()); - -// Invitation commands -program.addCommand(createInvitationCommand()); - // Memory commands — both as `me memory ` and top-level aliases (`me search`) program.addCommand(createMemoryCommand()); for (const c of createMemoryAliasCommands()) program.addCommand(c); @@ -103,12 +87,6 @@ program.addCommand(createCodexCommand()); // Local web UI program.addCommand(createServeCommand()); -// Engine-level RBAC commands (legacy; removed in 4E-CLI-7) -program.addCommand(createUserCommand()); -program.addCommand(createGrantCommand()); -program.addCommand(createRoleCommand()); -program.addCommand(createOwnerCommand()); - // Pack commands program.addCommand(createPackCommand()); diff --git a/packages/cli/serve/http-server.test.ts b/packages/cli/serve/http-server.test.ts index 95f627a..554e426 100644 --- a/packages/cli/serve/http-server.test.ts +++ b/packages/cli/serve/http-server.test.ts @@ -5,9 +5,10 @@ * upstream — no network access required. */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { SPACE_HEADER } from "@memory.build/protocol/headers"; import { - ENGINE_RPC_PATH, findAvailablePort, + MEMORY_RPC_PATH, type RunningServer, startHttpServer, } from "./http-server.ts"; @@ -72,8 +73,8 @@ describe("startHttpServer", () => { const servePort = await findAvailablePort("127.0.0.1", 34200); running = startHttpServer({ server: mock.url, - apiKey: "me.test.key", - engineSlug: "test-engine", + token: "sess-test-token", + space: "abc123def456", host: "127.0.0.1", port: servePort, }); @@ -127,18 +128,23 @@ describe("startHttpServer", () => { expect(json.result.ok).toBe(true); expect(mock.lastRequest?.method).toBe("POST"); - expect(mock.lastRequest?.path).toBe(ENGINE_RPC_PATH); + expect(mock.lastRequest?.path).toBe(MEMORY_RPC_PATH); expect(mock.lastRequest?.body).toBe(rpcBody); }); - test("/rpc injects Authorization: Bearer ", async () => { + test("/rpc injects Authorization: Bearer and X-Me-Space", async () => { await fetch(`${running.url}/rpc`, { method: "POST", headers: { "Content-Type": "application/json" }, body: "{}", }); - expect(mock.lastRequest?.headers.authorization).toBe("Bearer me.test.key"); + expect(mock.lastRequest?.headers.authorization).toBe( + "Bearer sess-test-token", + ); + expect(mock.lastRequest?.headers[SPACE_HEADER.toLowerCase()]).toBe( + "abc123def456", + ); }); test("/rpc surfaces upstream status codes", async () => { diff --git a/packages/cli/serve/http-server.ts b/packages/cli/serve/http-server.ts index 2b8cf1c..c7faa3c 100644 --- a/packages/cli/serve/http-server.ts +++ b/packages/cli/serve/http-server.ts @@ -3,23 +3,24 @@ * * Two concerns: * - * 1. Serve the web UI (placeholder HTML in step 2; embedded Vite build later). - * 2. Proxy `POST /rpc` to the configured engine's JSON-RPC endpoint with the - * stored API key injected (implemented in step 3). + * 1. Serve the embedded web UI (Vite build). + * 2. Proxy `POST /rpc` to the space memory JSON-RPC endpoint, injecting the + * session token (Authorization: Bearer) and the active space (X-Me-Space). * * The proxy is intentionally transparent — it forwards request bodies - * byte-for-byte and streams responses back, so any future `memory.*` RPC - * methods work without backend changes here. + * byte-for-byte and streams responses back, so any `memory.*` (and management) + * RPC methods work without backend changes here. */ +import { SPACE_HEADER } from "@memory.build/protocol/headers"; import { resolveAssetResponse } from "./web-assets.ts"; export interface ServeOptions { - /** Remote engine server URL (e.g., https://api.memory.build). */ + /** Remote server URL (e.g., https://api.memory.build). */ server: string; - /** API key for the active engine. Forwarded as Authorization: Bearer. */ - apiKey: string; - /** Active engine slug (for diagnostics / display). */ - engineSlug: string; + /** Session token. Forwarded as Authorization: Bearer. */ + token: string; + /** Active space slug. Forwarded as X-Me-Space. */ + space: string; /** Hostname to bind (defaults to 127.0.0.1). */ host: string; /** Port to bind. Use `findAvailablePort` first if you want auto-discovery. */ @@ -27,10 +28,10 @@ export interface ServeOptions { } /** - * Path on the remote server where the engine JSON-RPC endpoint lives. - * Kept as a constant so step 5's tests can assert the exact URL. + * Path on the remote server where the space memory JSON-RPC endpoint lives. + * Kept as a constant so tests can assert the exact URL. */ -export const ENGINE_RPC_PATH = "/api/v1/engine/rpc"; +export const MEMORY_RPC_PATH = "/api/v1/memory/rpc"; export interface RunningServer { /** The URL the server is listening on (e.g., http://127.0.0.1:3000). */ @@ -117,14 +118,15 @@ async function proxyRpc( req: Request, options: ServeOptions, ): Promise { - const targetUrl = new URL(ENGINE_RPC_PATH, options.server).toString(); + const targetUrl = new URL(MEMORY_RPC_PATH, options.server).toString(); // Rebuild the outgoing headers: drop hop-by-hop / host headers, keep - // Content-Type (the body needs it), and set our own Authorization. + // Content-Type (the body needs it), and set our own Authorization + space. const outHeaders = new Headers(); const contentType = req.headers.get("Content-Type"); if (contentType) outHeaders.set("Content-Type", contentType); - outHeaders.set("Authorization", `Bearer ${options.apiKey}`); + outHeaders.set("Authorization", `Bearer ${options.token}`); + outHeaders.set(SPACE_HEADER, options.space); outHeaders.set("Accept", "application/json"); try { diff --git a/packages/cli/util.test.ts b/packages/cli/util.test.ts index b405745..89c3eb0 100644 --- a/packages/cli/util.test.ts +++ b/packages/cli/util.test.ts @@ -1,41 +1,70 @@ /** * Unit tests for CLI utility helpers. * - * Tests resolution functions with mocked clients. + * Tests the space-model resolution functions with mocked clients. */ import { describe, expect, mock, test } from "bun:test"; -import type { EngineClient } from "@memory.build/client"; +import type { MemoryClient, UserClient } from "@memory.build/client"; -// We test the exported functions via dynamic import to avoid -// pulling in @clack/prompts at top level (it touches process.stdin). -const { resolveUserId } = await import("./util.ts"); +// Dynamic import to avoid pulling in @clack/prompts at top level (it touches +// process.stdin). +const { resolveSpacePrincipalId, resolveAgentId } = await import("./util.ts"); + +const UUID = "019d694f-79f6-7595-8faf-b70b01c11f98"; // ============================================================================= -// resolveUserId +// resolveSpacePrincipalId // ============================================================================= -describe("resolveUserId", () => { - test("returns UUID as-is when input is a valid UUIDv7", async () => { - const engine = {} as EngineClient; // should not be called - const id = "019d694f-79f6-7595-8faf-b70b01c11f98"; - const result = await resolveUserId(engine, id); - expect(result).toBe(id); +describe("resolveSpacePrincipalId", () => { + test("returns a UUIDv7 as-is without listing principals", async () => { + const memory = { + principal: { list: mock(() => Promise.reject(new Error("unused"))) }, + } as unknown as MemoryClient; + expect(await resolveSpacePrincipalId(memory, UUID, "text")).toBe(UUID); + expect(memory.principal.list).not.toHaveBeenCalled(); }); - test("resolves name via engine.user.getByName", async () => { - const engine = { - user: { - getByName: mock(() => + test("resolves a name via principal.list (with optional kind)", async () => { + const memory = { + principal: { + list: mock(() => Promise.resolve({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "alice", + principals: [{ id: UUID, kind: "g", name: "eng" }], }), ), }, - } as unknown as EngineClient; + } as unknown as MemoryClient; + + const id = await resolveSpacePrincipalId(memory, "eng", "text", "g"); + expect(id).toBe(UUID); + expect(memory.principal.list).toHaveBeenCalledWith({ kind: "g" }); + }); +}); + +// ============================================================================= +// resolveAgentId +// ============================================================================= + +describe("resolveAgentId", () => { + test("returns a UUIDv7 as-is without listing agents", async () => { + const user = { + agent: { list: mock(() => Promise.reject(new Error("unused"))) }, + } as unknown as UserClient; + expect(await resolveAgentId(user, UUID, "text")).toBe(UUID); + expect(user.agent.list).not.toHaveBeenCalled(); + }); + + test("resolves a name via agent.list", async () => { + const user = { + agent: { + list: mock(() => + Promise.resolve({ agents: [{ id: UUID, name: "bot" }] }), + ), + }, + } as unknown as UserClient; - const result = await resolveUserId(engine, "alice"); - expect(result).toBe("019d694f-79f6-7595-8faf-b70b01c11f98"); - expect(engine.user.getByName).toHaveBeenCalledWith({ name: "alice" }); + expect(await resolveAgentId(user, "bot", "text")).toBe(UUID); + expect(user.agent.list).toHaveBeenCalled(); }); }); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 2a0df46..6a22177 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -2,18 +2,13 @@ * Shared CLI utilities. * * Common patterns used across multiple command files: - * - Session token validation - * - Engine/API key validation - * - Org auto-resolution + * - Session / active-space validation + * - Memory / user client construction + * - Principal / agent resolution * - Error handling */ import * as clack from "@clack/prompts"; -import type { - AccountsClient, - EngineClient, - MemoryClient, - UserClient, -} from "./client.ts"; +import type { MemoryClient, UserClient } from "./client.ts"; import { createMemoryClient, createUserClient, RpcError } from "./client.ts"; import { clearSessionToken, type ResolvedCredentials } from "./credentials.ts"; import type { OutputFormat } from "./output.ts"; @@ -39,23 +34,6 @@ export function requireSession( } } -/** - * Ensure the user has an active engine with an API key. Exits with an error if not. - */ -export function requireEngine( - creds: ResolvedCredentials, - fmt: OutputFormat, -): asserts creds is ResolvedCredentials & { apiKey: string } { - if (!creds.apiKey) { - if (fmt === "text") { - clack.log.error("No active engine. Run 'me engine use' to select one."); - } else { - output({ error: "No active engine" }, fmt, () => {}); - } - process.exit(1); - } -} - /** * Ensure an active space (the X-Me-Space) is selected. Exits with an error if * not. Used by the space-scoped commands (memory, group, access, …). @@ -170,241 +148,6 @@ export async function resolveAgentId( process.exit(1); } -interface OrgInfo { - id: string; - name: string; - slug: string; -} - -/** - * Resolve an org from a flag, positional argument, or auto-resolution. - * - * Accepts a UUID, name, or slug. Falls back to auto-resolution if only one org. - * Priority: positionalArg > flagValue > auto-resolve (if exactly one org). - * Exits with an error if the org cannot be determined. - */ -export async function resolveOrg( - accounts: AccountsClient, - fmt: OutputFormat, - flagValue?: string, - positionalArg?: string, -): Promise { - const { orgs } = await accounts.org.list(); - const input = positionalArg ?? flagValue; - - if (input) { - // Try UUID match first - if (UUIDV7_RE.test(input)) { - const match = orgs.find((o) => o.id === input); - if (match) return match; - // Might be a valid org ID the user isn't a member of — use it as-is - return { id: input, name: input, slug: input }; - } - - // Match by name or slug (case-insensitive) - const lower = input.toLowerCase(); - const matches = orgs.filter( - (o) => o.name.toLowerCase() === lower || o.slug.toLowerCase() === lower, - ); - - if (matches.length === 1 && matches[0]) return matches[0]; - - if (matches.length === 0) { - const msg = `No organization found matching '${input}'.`; - if (fmt === "text") { - clack.log.error(msg); - if (orgs.length > 0) { - console.log(" Your organizations:"); - for (const org of orgs) { - console.log(` ${org.name} (${org.slug})`); - } - } - } else { - output({ error: msg, orgs }, fmt, () => {}); - } - process.exit(1); - } - - // Multiple matches (same name, different orgs) - const msg = `Multiple organizations match '${input}'. Use the org ID instead:`; - if (fmt === "text") { - clack.log.error(msg); - for (const org of matches) { - console.log(` ${org.name} (${org.slug}) — ${org.id}`); - } - } else { - output({ error: msg, orgs: matches }, fmt, () => {}); - } - process.exit(1); - } - - // Auto-resolve: pick if exactly one - if (orgs.length === 1 && orgs[0]) return orgs[0]; - - if (orgs.length === 0) { - const msg = "You don't belong to any organizations."; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - - // Multiple orgs — can't auto-resolve - const msg = - "You belong to multiple organizations. Use --org to specify which one."; - if (fmt === "text") { - clack.log.error(msg); - for (const org of orgs) { - console.log(` ${org.name} (${org.slug}) — ${org.id}`); - } - } else { - output( - { - error: msg, - orgs: orgs.map((o: { id: string; name: string; slug: string }) => ({ - id: o.id, - name: o.name, - slug: o.slug, - })), - }, - fmt, - () => {}, - ); - } - process.exit(1); -} - -/** - * Resolve an org ID from a flag, positional argument, or auto-resolution. - * - * Convenience wrapper around resolveOrg — returns just the ID. - */ -export async function resolveOrgId( - accounts: AccountsClient, - fmt: OutputFormat, - flagValue?: string, - positionalArg?: string, -): Promise { - const org = await resolveOrg(accounts, fmt, flagValue, positionalArg); - return org.id; -} - -/** - * Resolve a user or role by ID or name. If the argument looks like a UUIDv7, - * fetches by ID; otherwise fetches by name. Returns the UUID. - */ -export async function resolveUserId( - engine: EngineClient, - idOrName: string, -): Promise { - if (UUIDV7_RE.test(idOrName)) return idOrName; - const user = await engine.user.getByName({ name: idOrName }); - return user.id; -} - -/** - * Resolve an identity ID from an email, name, or UUID. - * - * - UUID: used as-is - * - Email (contains @): looked up via identity.getByEmail - * - Otherwise: error with guidance - */ -export async function resolveIdentityId( - accounts: AccountsClient, - fmt: OutputFormat, - input: string, -): Promise { - if (UUIDV7_RE.test(input)) return input; - - if (input.includes("@")) { - const { identity } = await accounts.identity.getByEmail({ email: input }); - if (identity) return identity.id; - - const msg = `No identity found with email '${input}'. They may need to sign up first, or use 'me invitation create' to invite them.`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - - const msg = `'${input}' is not a valid ID or email. Provide a UUID or email address.`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); -} - -/** - * Resolve an identity from an org's member list by name, email, or UUID. - * - * Used for operations on existing members (e.g., remove). - */ -export async function resolveMember( - accounts: AccountsClient, - fmt: OutputFormat, - orgId: string, - input: string, -): Promise<{ identityId: string; name: string; email: string }> { - const { members } = await accounts.org.member.list({ orgId }); - - // UUID match - if (UUIDV7_RE.test(input)) { - const match = members.find((m) => m.identityId === input); - if (match) return match; - // UUID not in member list — return it as-is (server will error if invalid) - return { identityId: input, name: input, email: "" }; - } - - // Email match - if (input.includes("@")) { - const lower = input.toLowerCase(); - const match = members.find((m) => m.email.toLowerCase() === lower); - if (match) return match; - - const msg = `No member with email '${input}' in this organization.`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - - // Name match - const lower = input.toLowerCase(); - const matches = members.filter((m) => m.name.toLowerCase() === lower); - - if (matches.length === 1 && matches[0]) return matches[0]; - - if (matches.length === 0) { - const msg = `No member named '${input}' in this organization.`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - - // Multiple matches - const msg = `Multiple members named '${input}'. Use their email instead:`; - if (fmt === "text") { - clack.log.error(msg); - for (const m of matches) { - console.log(` ${m.name} — ${m.email}`); - } - } else { - output({ error: msg, members: matches }, fmt, () => {}); - } - process.exit(1); -} - /** * Detect an authentication error from the server. * From dd6f185babfb38a368c80ecea628553f47598b85 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 18:23:25 +0200 Subject: [PATCH 059/156] docs(todo): note me apikey create UX when agent not in space Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/TODO.md b/TODO.md index b676bbf..075ecbc 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,18 @@ implies users can mint their own keys. `member == self` (a user) in addition to agents the caller owns. Weigh against the "humans use sessions only" security stance. +## CLI: `me apikey create ` when the agent isn't in the space + +`apiKey.create` requires the agent already be a member of the active space +(`requireOwnedAgent` → NOT_FOUND otherwise). `me apikey create ` surfaces +that raw NOT_FOUND, so the user has to know to run `me agent add ` first. + +- [ ] Improve the UX: either pre-check membership in `me apikey create` and emit + an actionable hint ("agent X isn't in this space — run `me agent add X`"), + or offer to add it (self-service `principal.add`, which is already allowed + for your own agent) before minting the key. Map the server NOT_FOUND to the + friendlier message at minimum. + ## Space invitations The CLI spec includes `me space invite` / `invite list` / `invite revoke` From edc64d423b07e11162d2a221d65af98cf74e2fff Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 21:25:22 +0200 Subject: [PATCH 060/156] feat(server): remove legacy engine + accounts RPC endpoints (5A) Switch off the legacy server surface. The new model serves only /api/v1/memory/rpc, /api/v1/user/rpc, and the auth device-flow. - router: drop the /engine/rpc + /accounts/rpc routes and their handlers; rpc/index no longer exports engineMethods/accountsMethods; delete rpc/engine/* and rpc/accounts/*. - middleware/authenticate.ts reduced to extractBearerToken (the shared helper authenticate-space/user rely on); legacy auth funcs/types dropped from the middleware + lib barrels. - index.ts: remove both Bun.SQL pools (accountsSql/engineSql), accountsDb, and the accounts + per-engine boot-migrate; drop ACCOUNTS_* env. The single postgres.js `db` (+ workerDb) now drive everything; readyHandler checks it. - context.ts: drop accountsDb/accountsSql/engineSql. Test parity: authenticate.test keeps extractBearerToken; router/wiring/server integration tests repointed to the memory/user endpoints (adding the previously-missing /memory/rpc + /user/rpc route-match, missing-X-Me-Space 400, and whoami happy-path cases). Deleted org/invitation/engine tests have no new equivalent (concepts removed or covered by the rpc/memory suite). Net -6647 lines. typecheck + lint + unit (886) + server integration (50) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/server/context.ts | 10 +- packages/server/handlers/health.ts | 27 +- packages/server/index.ts | 172 +--- packages/server/lib.ts | 10 +- .../server/middleware/authenticate.test.ts | 270 +----- packages/server/middleware/authenticate.ts | 271 +----- .../server/middleware/client-version.test.ts | 2 +- packages/server/middleware/index.ts | 14 +- packages/server/router.test.ts | 36 +- packages/server/router.ts | 76 +- .../rpc/accounts/engine.integration.test.ts | 642 ------------- packages/server/rpc/accounts/engine.ts | 432 --------- packages/server/rpc/accounts/index.ts | 41 - packages/server/rpc/accounts/invitation.ts | 197 ---- packages/server/rpc/accounts/me.test.ts | 137 --- packages/server/rpc/accounts/me.ts | 57 -- packages/server/rpc/accounts/org-member.ts | 225 ----- .../rpc/accounts/org.integration.test.ts | 156 ---- packages/server/rpc/accounts/org.ts | 204 ----- packages/server/rpc/accounts/schemas.test.ts | 513 ----------- packages/server/rpc/accounts/schemas.ts | 77 -- packages/server/rpc/accounts/session.ts | 45 - packages/server/rpc/accounts/types.ts | 72 -- packages/server/rpc/engine/api-key.ts | 159 ---- packages/server/rpc/engine/grant.test.ts | 253 ------ packages/server/rpc/engine/grant.ts | 161 ---- packages/server/rpc/engine/index.ts | 35 - packages/server/rpc/engine/memory.test.ts | 249 ------ packages/server/rpc/engine/memory.ts | 414 --------- packages/server/rpc/engine/owner.test.ts | 194 ---- packages/server/rpc/engine/owner.ts | 127 --- packages/server/rpc/engine/role.ts | 180 ---- packages/server/rpc/engine/schemas.test.ts | 842 ------------------ packages/server/rpc/engine/schemas.ts | 106 --- packages/server/rpc/engine/types.ts | 62 -- packages/server/rpc/engine/user.ts | 177 ---- packages/server/rpc/index.ts | 8 - packages/server/server.integration.test.ts | 53 +- packages/server/wiring.test.ts | 185 +--- 39 files changed, 122 insertions(+), 6769 deletions(-) delete mode 100644 packages/server/rpc/accounts/engine.integration.test.ts delete mode 100644 packages/server/rpc/accounts/engine.ts delete mode 100644 packages/server/rpc/accounts/index.ts delete mode 100644 packages/server/rpc/accounts/invitation.ts delete mode 100644 packages/server/rpc/accounts/me.test.ts delete mode 100644 packages/server/rpc/accounts/me.ts delete mode 100644 packages/server/rpc/accounts/org-member.ts delete mode 100644 packages/server/rpc/accounts/org.integration.test.ts delete mode 100644 packages/server/rpc/accounts/org.ts delete mode 100644 packages/server/rpc/accounts/schemas.test.ts delete mode 100644 packages/server/rpc/accounts/schemas.ts delete mode 100644 packages/server/rpc/accounts/session.ts delete mode 100644 packages/server/rpc/accounts/types.ts delete mode 100644 packages/server/rpc/engine/api-key.ts delete mode 100644 packages/server/rpc/engine/grant.test.ts delete mode 100644 packages/server/rpc/engine/grant.ts delete mode 100644 packages/server/rpc/engine/index.ts delete mode 100644 packages/server/rpc/engine/memory.test.ts delete mode 100644 packages/server/rpc/engine/memory.ts delete mode 100644 packages/server/rpc/engine/owner.test.ts delete mode 100644 packages/server/rpc/engine/owner.ts delete mode 100644 packages/server/rpc/engine/role.ts delete mode 100644 packages/server/rpc/engine/schemas.test.ts delete mode 100644 packages/server/rpc/engine/schemas.ts delete mode 100644 packages/server/rpc/engine/types.ts delete mode 100644 packages/server/rpc/engine/user.ts diff --git a/packages/server/context.ts b/packages/server/context.ts index 7c00b80..1223165 100644 --- a/packages/server/context.ts +++ b/packages/server/context.ts @@ -1,8 +1,6 @@ -import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { CoreStore } from "@memory.build/engine/core"; -import type { SQL } from "bun"; import type { Sql } from "postgres"; /** @@ -10,13 +8,7 @@ import type { Sql } from "postgres"; * Passed to createRouter() at startup. */ export interface ServerContext { - /** Accounts database operations (legacy: org/engine, until Phase 5) */ - accountsDb: AccountsDB; - /** Accounts database pool (for health checks) */ - accountsSql: SQL; - /** Engine database pool (EngineDB created per-request based on slug) */ - engineSql: SQL; - /** New-model pool (postgres.js): auth + core + per-space schemas, one DB */ + /** Pool (postgres.js): auth + core + per-space schemas, one DB */ db: Sql; /** Auth store (auth schema): me/session/identity/device + OAuth accounts */ auth: AuthStore; diff --git a/packages/server/handlers/health.ts b/packages/server/handlers/health.ts index 1e3c061..c2d3bdf 100644 --- a/packages/server/handlers/health.ts +++ b/packages/server/handlers/health.ts @@ -1,5 +1,5 @@ import { info } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; +import type { Sql } from "postgres"; import { json, text } from "../util/response"; /** @@ -14,32 +14,23 @@ export function healthHandler(_request: Request): Response { /** * Readiness check handler. - * Verifies both database pools are alive via SELECT 1. - * Returns 200 if both succeed, 503 if either fails. + * Verifies the database pool is alive via SELECT 1. + * Returns 200 on success, 503 on failure. */ export function readyHandler( - accountsSql: SQL, - engineSql: SQL, + db: Sql, ): (_request: Request) => Promise { return async (_request: Request) => { const checks: Record = {}; - const [accountsResult, engineResult] = await Promise.allSettled([ - accountsSql`SELECT 1`, - engineSql`SELECT 1`, - ]); + const [dbResult] = await Promise.allSettled([db`SELECT 1`]); - checks.accounts_db = - accountsResult.status === "fulfilled" + checks.db = + dbResult.status === "fulfilled" ? "ok" - : `error: ${accountsResult.reason instanceof Error ? accountsResult.reason.message : String(accountsResult.reason)}`; + : `error: ${dbResult.reason instanceof Error ? dbResult.reason.message : String(dbResult.reason)}`; - checks.engine_db = - engineResult.status === "fulfilled" - ? "ok" - : `error: ${engineResult.reason instanceof Error ? engineResult.reason.message : String(engineResult.reason)}`; - - const allOk = checks.accounts_db === "ok" && checks.engine_db === "ok"; + const allOk = checks.db === "ok"; return json( { diff --git a/packages/server/index.ts b/packages/server/index.ts index b56c4ae..361787a 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -1,6 +1,4 @@ // packages/server/index.ts -import { createAccountsDB } from "@memory.build/accounts"; -import { migrate as migrateAccounts } from "@memory.build/accounts/migrate/runner"; import { authStore } from "@memory.build/auth"; import { bootstrapSpaceDatabase, @@ -10,12 +8,6 @@ import { } from "@memory.build/database"; import type { EmbeddingConfig } from "@memory.build/embedding"; import { coreStore } from "@memory.build/engine/core"; -import { - discoverEngineSchemas, - slugToSchema, -} from "@memory.build/engine/migrate"; -import { bootstrap as bootstrapEngine } from "@memory.build/engine/migrate/bootstrap"; -import { migrateAll as migrateEngines } from "@memory.build/engine/migrate/runner"; import { DEFAULT_WORKER_TIMEOUTS, WorkerPool, @@ -70,28 +62,18 @@ configure({ // ============================================================================= // // Required: -// ACCOUNTS_DATABASE_URL - PostgreSQL connection string for accounts database -// (stores engines, API keys, users) -// ENGINE_DATABASE_URL - PostgreSQL connection string for engine databases -// (stores memories, each engine in its own schema) +// ENGINE_DATABASE_URL - PostgreSQL connection string for the database that +// holds the auth + core control plane and every +// per-space me_ schema (one DB, one pool) // API_BASE_URL - Public URL for OAuth callbacks // (e.g., "https://memory.build") // // Optional: // PORT - HTTP server port (default: 3000) -// ACCOUNTS_SCHEMA - Schema name in accounts database (default: "accounts") -// -// Connection Pool - Accounts Database: -// ACCOUNTS_POOL_MAX - Max connections (default: 10) -// ACCOUNTS_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: 300) -// ACCOUNTS_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: 0) -// ACCOUNTS_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: 30) -// ACCOUNTS_STATEMENT_TIMEOUT - Per-accounts-query timeout (default: 25s) -// ACCOUNTS_LOCK_TIMEOUT - Per-accounts-lock wait timeout (default: 5s) -// ACCOUNTS_TRANSACTION_TIMEOUT - Per-accounts-transaction timeout (default: 30s) -// ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Idle-in-transaction timeout (default: 30s) +// AUTH_SCHEMA - Auth schema name (default: "auth") +// CORE_SCHEMA - Core control-plane schema name (default: "core") // -// Connection Pool - Engine Database: +// Connection Pool - Database: // ENGINE_POOL_MAX - Max connections (default: 20) // ENGINE_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: 300) // ENGINE_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: 0) @@ -146,11 +128,6 @@ function parseIntEnv( const port = process.env.PORT || 3000; -const accountsDatabaseUrl = process.env.ACCOUNTS_DATABASE_URL; -if (!accountsDatabaseUrl) { - throw new Error("ACCOUNTS_DATABASE_URL environment variable is required"); -} - const engineDatabaseUrl = process.env.ENGINE_DATABASE_URL; if (!engineDatabaseUrl) { throw new Error("ENGINE_DATABASE_URL environment variable is required"); @@ -164,9 +141,7 @@ if (!apiBaseUrl) { const deviceFlowCleanupCron = process.env.DEVICE_FLOW_CLEANUP_CRON || "*/15 * * * *"; -const accountsSchema = process.env.ACCOUNTS_SCHEMA || "accounts"; - -// New-model schema names (single DB, postgres.js pool): auth + core control plane. +// Schema names (single DB, postgres.js pool): auth + core control plane. const authSchema = process.env.AUTH_SCHEMA || "auth"; const coreSchema = process.env.CORE_SCHEMA || "core"; @@ -176,29 +151,7 @@ const workerCount = parseIntEnv( "2", ); -// Connection pool settings - Accounts database -const accountsPoolMax = parseIntEnv( - "ACCOUNTS_POOL_MAX", - process.env.ACCOUNTS_POOL_MAX || "", - "10", -); -const accountsPoolIdleReapSeconds = parseIntEnv( - "ACCOUNTS_POOL_IDLE_REAP_SECONDS", - process.env.ACCOUNTS_POOL_IDLE_REAP_SECONDS || "", - "300", -); -const accountsPoolMaxLifetime = parseIntEnv( - "ACCOUNTS_POOL_MAX_LIFETIME", - process.env.ACCOUNTS_POOL_MAX_LIFETIME || "", - "0", -); -const accountsPoolConnectionTimeout = parseIntEnv( - "ACCOUNTS_POOL_CONNECTION_TIMEOUT", - process.env.ACCOUNTS_POOL_CONNECTION_TIMEOUT || "", - "30", -); - -// Connection pool settings - Engine database +// Connection pool settings - database const enginePoolMax = parseIntEnv( "ENGINE_POOL_MAX", process.env.ENGINE_POOL_MAX || "", @@ -343,23 +296,8 @@ if (configuredProviders.length === 0) { // Database Pools // ============================================================================= -// Create database connection pools -const accountsSql = new Bun.SQL(accountsDatabaseUrl, { - max: accountsPoolMax, - idleTimeout: accountsPoolIdleReapSeconds, - maxLifetime: accountsPoolMaxLifetime, - connectionTimeout: accountsPoolConnectionTimeout, -}); - -const engineSql = new Bun.SQL(engineDatabaseUrl, { - max: enginePoolMax, - idleTimeout: enginePoolIdleReapSeconds, - maxLifetime: enginePoolMaxLifetime, - connectionTimeout: enginePoolConnectionTimeout, -}); - -// Dedicated worker pool (postgres.js) on the new-model DB — the embedding -// worker processes the per-space me_ schemas that live there. +// Dedicated worker pool (postgres.js) — the embedding worker processes the +// per-space me_ schemas. const workerDb = postgres(workerEngineDatabaseUrl, { max: workerEnginePoolMax, idle_timeout: workerEnginePoolIdleReapSeconds, @@ -368,9 +306,8 @@ const workerDb = postgres(workerEngineDatabaseUrl, { onnotice: () => {}, }); -// New-model pool (postgres.js): the auth + core control plane and the per-space -// me_ data schemas all live in one database, one pool. The legacy Bun.SQL -// accountsSql/engineSql pools above stay until Phase 5 removes the old paths. +// The single application pool (postgres.js): the auth + core control plane and +// the per-space me_ data schemas all live in one database, one pool. const db = postgres(engineDatabaseUrl, { max: enginePoolMax, idle_timeout: enginePoolIdleReapSeconds, @@ -379,10 +316,7 @@ const db = postgres(engineDatabaseUrl, { onnotice: () => {}, }); -// Create accounts DB with operations layer -const accountsDb = createAccountsDB(accountsSql, accountsSchema); - -// Auth store (auth schema) on the new-model postgres.js pool. +// Auth store (auth schema) on the application pool. const auth = authStore(db, authSchema); // Core control-plane store (core schema) on the same pool. @@ -392,77 +326,10 @@ const core = coreStore(db, coreSchema); // Database Bootstrap & Migrations (blocking — server won't serve until current) // ============================================================================= -// Bootstrap engine database (extensions + roles, idempotent) -// If the DB user lacks CREATE EXTENSION privileges (e.g., RDS), this will -// throw with a clear error describing what's missing. -await bootstrapEngine(engineSql); -info("Engine database bootstrapped"); - -// Migrate accounts schema (scaffold creates schema if missing) -const accountsMigrateResult = await migrateAccounts( - accountsSql, - { schema: accountsSchema }, - SERVER_VERSION, -); -if (accountsMigrateResult.status === "error") { - throw new Error( - `Accounts migration failed: ${accountsMigrateResult.error?.message}`, - ); -} -if (accountsMigrateResult.applied.length > 0) { - info("Accounts migrations applied", { - applied: accountsMigrateResult.applied, - }); -} else { - info("Accounts schema up to date"); -} - -// Migrate all engine schemas -const engineSchemas = await discoverEngineSchemas(engineSql); -if (engineSchemas.length > 0) { - const engineMigrateResults = await migrateEngines( - engineSql, - engineSchemas, - { embedding_dimensions: embeddingConstants.dimensions }, - SERVER_VERSION, - ); - - let totalApplied = 0; - let totalErrors = 0; - for (const [schema, result] of engineMigrateResults) { - if (result.status === "error") { - totalErrors++; - reportError( - `Engine migration failed for ${schema}`, - result.error ?? new Error("Unknown migration error"), - ); - } else if (result.applied.length > 0) { - totalApplied += result.applied.length; - } - } - - if (totalErrors > 0) { - throw new Error( - `${totalErrors} engine schema(s) failed to migrate. Check logs for details.`, - ); - } - - if (totalApplied > 0) { - info("Engine migrations applied", { - schemas: engineSchemas.length, - totalApplied, - }); - } else { - info("Engine schemas up to date", { schemas: engineSchemas.length }); - } -} else { - info("No engine schemas to migrate"); -} - -// New model (Phase 4 cutover): prepare the DB for per-space schemas and migrate -// the auth + core control-plane schemas on the single postgres.js pool. These -// run alongside the legacy schemas above; the new auth/memory paths consume -// them as they come online (4B+). +// Prepare the database for per-space schemas (extensions + roles, idempotent) +// and migrate the auth + core control-plane schemas on the application pool. +// If the DB user lacks CREATE EXTENSION privileges (e.g., RDS), bootstrap +// throws with a clear error describing what's missing. await bootstrapSpaceDatabase(db); await migrateCore(db); await migrateAuth(db); @@ -473,9 +340,6 @@ info("Core + auth schemas migrated"); // ============================================================================= const serverContext: ServerContext = { - accountsDb, - accountsSql, - engineSql, db, auth, core, @@ -618,8 +482,6 @@ async function shutdown() { // Close database pools try { - await accountsSql.close(); - await engineSql.close(); await workerDb.end(); await db.end(); } catch (error) { diff --git a/packages/server/lib.ts b/packages/server/lib.ts index 87c6d48..cb5666c 100644 --- a/packages/server/lib.ts +++ b/packages/server/lib.ts @@ -2,13 +2,5 @@ // Type exports for consumers who need to test or extend the server export type { ServerContext } from "./context"; -export { - type AccountsAuthContext, - type AuthContext, - type AuthResult, - authenticateAccounts, - authenticateEngine, - ENGINE_SCHEMA_PREFIX, - type EngineAuthContext, -} from "./middleware"; +export { extractBearerToken } from "./middleware"; export { createRouter, type Router } from "./router"; diff --git a/packages/server/middleware/authenticate.test.ts b/packages/server/middleware/authenticate.test.ts index b485c6a..2041edc 100644 --- a/packages/server/middleware/authenticate.test.ts +++ b/packages/server/middleware/authenticate.test.ts @@ -1,16 +1,5 @@ -import { describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; -import type { AuthStore } from "@memory.build/auth"; -import type { EngineDB } from "@memory.build/engine"; -import type { SQL } from "bun"; -import { - authenticateAccounts, - authenticateEngine, - type CreateEngineDBFn, - type EngineInfo, - extractBearerToken, - type Identity, -} from "./authenticate"; +import { describe, expect, test } from "bun:test"; +import { extractBearerToken } from "./authenticate"; describe("extractBearerToken", () => { test("extracts token from valid Authorization header", () => { @@ -39,258 +28,3 @@ describe("extractBearerToken", () => { expect(extractBearerToken(request)).toBeNull(); }); }); - -describe("authenticateAccounts", () => { - const mockIdentity: Identity = { - id: "identity-123", - email: "test@example.com", - name: "Test User", - }; - - // A validate_session row: the session plus its user. - const validatedSession = { - sessionId: "session-1", - userId: mockIdentity.id, - email: mockIdentity.email, - name: mockIdentity.name, - expiresAt: new Date("2026-12-31T00:00:00Z"), - }; - - test("returns 401 when no Authorization header", async () => { - const request = new Request("http://localhost/test"); - const mockAuth = { - validateSession: mock(() => Promise.resolve(null)), - } as unknown as AuthStore; - - const result = await authenticateAccounts(request, mockAuth); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 401 when session validation fails", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: "Bearer invalid-token" }, - }); - const validateSession = mock(() => Promise.resolve(null)); - const mockAuth = { validateSession } as unknown as AuthStore; - - const result = await authenticateAccounts(request, mockAuth); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - expect(validateSession).toHaveBeenCalledWith("invalid-token"); - }); - - test("returns identity when session is valid", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: "Bearer valid-token" }, - }); - const mockAuth = { - validateSession: mock(() => Promise.resolve(validatedSession)), - } as unknown as AuthStore; - - const result = await authenticateAccounts(request, mockAuth); - - expect(result.ok).toBe(true); - if (result.ok && result.context.type === "accounts") { - expect(result.context.identity).toEqual(mockIdentity); - } - }); -}); - -describe("authenticateEngine", () => { - const mockEngine: EngineInfo = { - id: "engine-123", - orgId: "org-456", - slug: "abc123xyz789", - name: "Test Engine", - shardId: 7, - status: "active", - }; - - const createMockAccountsDb = (engine: EngineInfo | null) => - ({ - getEngineBySlug: mock(() => Promise.resolve(engine)), - }) as unknown as AccountsDB; - - const createMockEngineDb = (validationResult: { - valid: boolean; - userId?: string; - apiKeyId?: string; - }) => - ({ - validateApiKey: mock(() => Promise.resolve(validationResult)), - setUser: mock(() => {}), - }) as unknown as EngineDB; - - const mockCreateEngineDB = mock((_sql: SQL, _schema: string) => { - return createMockEngineDb({ - valid: true, - userId: "user-789", - apiKeyId: "apikey-abc", - }); - }) as unknown as CreateEngineDBFn; - - // Valid API key format: me.{slug}.{lookupId}.{secret} - // Secret must be exactly 32 chars (base64url) - const validApiKey = - "me.abc123xyz789.Sh00uLs5rmSHHun3.pREy3xfnbCpgUXiaBcDefghij1234567"; - - test("returns 401 when no Authorization header", async () => { - const request = new Request("http://localhost/test"); - const mockAccountsDb = createMockAccountsDb(mockEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 401 when API key format is invalid", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: "Bearer invalid-format" }, - }); - const mockAccountsDb = createMockAccountsDb(mockEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 401 when engine not found", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const mockAccountsDb = createMockAccountsDb(null); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 403 when engine is suspended", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const suspendedEngine = { ...mockEngine, status: "suspended" as const }; - const mockAccountsDb = createMockAccountsDb(suspendedEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(403); - } - }); - - test("returns 403 when engine is deleted", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const deletedEngine = { ...mockEngine, status: "deleted" as const }; - const mockAccountsDb = createMockAccountsDb(deletedEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(403); - } - }); - - test("returns 401 when API key validation fails", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const mockAccountsDb = createMockAccountsDb(mockEngine); - const mockCreateEngineDBInvalid = mock(() => - createMockEngineDb({ valid: false }), - ) as unknown as CreateEngineDBFn; - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDBInvalid, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns engine context when authentication succeeds", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const mockAccountsDb = createMockAccountsDb(mockEngine); - const mockEngineDb = createMockEngineDb({ - valid: true, - userId: "user-789", - apiKeyId: "apikey-abc", - }); - const mockCreateEngineDBSuccess = mock( - () => mockEngineDb, - ) as unknown as CreateEngineDBFn; - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDBSuccess, - ); - - expect(result.ok).toBe(true); - if (result.ok && result.context.type === "engine") { - expect(result.context.db).toBe(mockEngineDb); - expect(result.context.userId).toBe("user-789"); - expect(result.context.apiKeyId).toBe("apikey-abc"); - expect(result.context.engine).toEqual(mockEngine); - expect(mockEngineDb.setUser).toHaveBeenCalledWith("user-789"); - expect(mockCreateEngineDBSuccess).toHaveBeenCalledWith( - {} as SQL, - "me_abc123xyz789", - { shard: 7 }, - ); - } - }); -}); diff --git a/packages/server/middleware/authenticate.ts b/packages/server/middleware/authenticate.ts index fbe2470..6772a55 100644 --- a/packages/server/middleware/authenticate.ts +++ b/packages/server/middleware/authenticate.ts @@ -1,94 +1,11 @@ -import type { AccountsDB } from "@memory.build/accounts"; -import type { AuthStore } from "@memory.build/auth"; -import { - createEngineDB, - type EngineDB, - parseApiKey, -} from "@memory.build/engine"; -import { debug, span } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import { forbidden, unauthorized } from "../util/response"; - -// ============================================================================= -// Constants -// ============================================================================= - -/** - * Schema prefix for engine databases. - * Engine schemas are named `{ENGINE_SCHEMA_PREFIX}{engineSlug}`. - */ -export const ENGINE_SCHEMA_PREFIX = "me_"; - -// ============================================================================= -// Types -// ============================================================================= - -/** - * The authenticated principal for accounts RPC — the session-validated user - * (auth.users), kept lightweight; me.get fetches the full record when needed. - */ -export interface Identity { - id: string; - email: string; - name: string; -} - -/** - * Engine info from accounts DB. - */ -export interface EngineInfo { - id: string; - orgId: string; - slug: string; - name: string; - shardId: number; - status: "active" | "suspended" | "deleted"; -} - /** - * Auth context for accounts RPC requests. - */ -export interface AccountsAuthContext { - type: "accounts"; - identity: Identity; -} - -/** - * Auth context for engine RPC requests. - */ -export interface EngineAuthContext { - type: "engine"; - db: EngineDB; - userId: string; - apiKeyId: string; - engine: EngineInfo; -} - -/** - * Factory function type for creating EngineDB instances. - * Allows dependency injection for testing. - */ -export type CreateEngineDBFn = ( - sql: SQL, - schema: string, - options?: { shard?: number }, -) => EngineDB; - -/** - * Union type for all auth contexts. - */ -export type AuthContext = AccountsAuthContext | EngineAuthContext; - -/** - * Result of authentication attempt. + * Shared bearer-token extraction for the RPC auth middlewares. + * + * The per-endpoint authenticators live in `authenticate-space.ts` (memory RPC: + * session or api key + X-Me-Space) and `authenticate-user.ts` (user RPC: + * session only). Both resolve the credential from the `Authorization` header + * via `extractBearerToken`. */ -export type AuthResult = - | { ok: true; context: AuthContext } - | { ok: false; error: Response }; - -// ============================================================================= -// Helpers -// ============================================================================= /** * Extract Bearer token from Authorization header. @@ -112,179 +29,3 @@ export function extractBearerToken(request: Request): string | null { return token; } - -// ============================================================================= -// Accounts Authentication -// ============================================================================= - -/** - * Authenticate request for accounts RPC. - * Validates the session token (auth schema) and returns the user as identity. - */ -export async function authenticateAccounts( - request: Request, - auth: AuthStore, -): Promise { - const token = extractBearerToken(request); - if (!token) { - debug("accounts auth failed: missing Authorization header"); - return { - ok: false, - error: unauthorized("Missing or invalid Authorization header"), - }; - } - - const session = await auth.validateSession(token); - if (!session) { - debug("accounts auth failed: invalid or expired session"); - return { - ok: false, - error: unauthorized("Invalid or expired session"), - }; - } - - debug("accounts auth succeeded", { userId: session.userId }); - return { - ok: true, - context: { - type: "accounts", - identity: { - id: session.userId, - email: session.email, - name: session.name, - }, - }, - }; -} - -// ============================================================================= -// Engine Authentication -// ============================================================================= - -/** - * Authenticate request for engine RPC. - * Parses API key, looks up engine, validates key against engine DB. - * - * Security note: Error messages are intentionally generic to prevent - * enumeration attacks. The specific failure reason is logged for debugging. - */ -export async function authenticateEngine( - request: Request, - accountsDb: AccountsDB, - engineSql: SQL, - createEngineDBFn: CreateEngineDBFn = createEngineDB, -): Promise { - return span("auth.engine", { - attributes: { - "auth.type": "engine", - }, - callback: () => - authenticateEngineInner(request, accountsDb, engineSql, createEngineDBFn), - }); -} - -async function authenticateEngineInner( - request: Request, - accountsDb: AccountsDB, - engineSql: SQL, - createEngineDBFn: CreateEngineDBFn, -): Promise { - // 1. Extract bearer token - const token = extractBearerToken(request); - if (!token) { - debug("engine auth failed: missing Authorization header"); - return { - ok: false, - error: unauthorized("Missing or invalid Authorization header"), - }; - } - - // 2. Parse API key - const parsed = parseApiKey(token); - if (!parsed) { - debug("engine auth failed: invalid API key format"); - return { ok: false, error: unauthorized("Invalid API key") }; - } - - const { engineSlug, lookupId, secret } = parsed; - - // 3. Look up engine in accounts DB - const engine = await span("auth.engine.accounts_lookup", { - attributes: { - "engine.slug": engineSlug, - "api_key.lookup_id": lookupId, - }, - callback: () => accountsDb.getEngineBySlug(engineSlug), - }); - if (!engine) { - // Generic error to prevent engine enumeration - debug("engine auth failed: engine not found", { engineSlug }); - return { ok: false, error: unauthorized("Invalid API key") }; - } - - // 4. Check engine status - if (engine.status !== "active") { - // 403 Forbidden for suspended/deleted engines - the key is valid but access is denied - debug("engine auth failed: engine not active", { - engineSlug, - status: engine.status, - }); - return { - ok: false, - error: forbidden("Access denied"), - }; - } - - // 5. Create EngineDB for this engine's schema - const schema = `${ENGINE_SCHEMA_PREFIX}${engineSlug}`; - const db = createEngineDBFn(engineSql, schema, { shard: engine.shardId }); - - // 6. Validate API key - const validation = await span("auth.engine.validate_api_key", { - attributes: { - "db.schema": schema, - "engine.id": engine.id, - "engine.slug": engineSlug, - "engine.shard": engine.shardId, - "api_key.lookup_id": lookupId, - }, - callback: () => db.validateApiKey(lookupId, secret), - }); - if (!validation.valid || !validation.userId || !validation.apiKeyId) { - debug("engine auth failed: API key validation failed", { - engineSlug, - lookupId, - }); - return { ok: false, error: unauthorized("Invalid API key") }; - } - - // 7. Set user on db for RLS context - db.setUser(validation.userId); - - // 8. Build engine info - const engineInfo: EngineInfo = { - id: engine.id, - orgId: engine.orgId, - slug: engine.slug, - name: engine.name, - shardId: engine.shardId, - status: engine.status, - }; - - debug("engine auth succeeded", { - engineSlug, - userId: validation.userId, - apiKeyId: validation.apiKeyId, - }); - - return { - ok: true, - context: { - type: "engine", - db, - userId: validation.userId, - apiKeyId: validation.apiKeyId, - engine: engineInfo, - }, - }; -} diff --git a/packages/server/middleware/client-version.test.ts b/packages/server/middleware/client-version.test.ts index c9570be..0a24923 100644 --- a/packages/server/middleware/client-version.test.ts +++ b/packages/server/middleware/client-version.test.ts @@ -5,7 +5,7 @@ import { checkClientVersion } from "./client-version"; const MIN = "0.2.0"; function req(headers: Record = {}): Request { - return new Request("http://localhost/api/v1/engine/rpc", { + return new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers, }); diff --git a/packages/server/middleware/index.ts b/packages/server/middleware/index.ts index d5285d0..78a0d48 100644 --- a/packages/server/middleware/index.ts +++ b/packages/server/middleware/index.ts @@ -1,16 +1,4 @@ -export { - type AccountsAuthContext, - type AuthContext, - type AuthResult, - authenticateAccounts, - authenticateEngine, - type CreateEngineDBFn, - ENGINE_SCHEMA_PREFIX, - type EngineAuthContext, - type EngineInfo, - extractBearerToken, - type Identity, -} from "./authenticate"; +export { extractBearerToken } from "./authenticate"; export { authenticateSpace, SPACE_HEADER, diff --git a/packages/server/router.test.ts b/packages/server/router.test.ts index 8fc96e9..fce1db2 100644 --- a/packages/server/router.test.ts +++ b/packages/server/router.test.ts @@ -1,9 +1,7 @@ import { describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { CoreStore } from "@memory.build/engine/core"; -import type { SQL } from "bun"; import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; @@ -12,12 +10,6 @@ import { createRouter } from "./router"; // Mock ServerContext for testing function createMockContext(): ServerContext { return { - accountsDb: { - validateSession: mock(() => Promise.resolve(null)), - getEngineBySlug: mock(() => Promise.resolve(null)), - } as unknown as AccountsDB, - accountsSql: {} as SQL, - engineSql: {} as SQL, db: {} as Sql, auth: { validateSession: mock(() => Promise.resolve(null)), @@ -113,30 +105,30 @@ describe("matchRoute", () => { }); }); - describe("accounts RPC endpoint", () => { - test("matches POST /api/v1/accounts/rpc", () => { - const match = router.matchRoute("POST", "/api/v1/accounts/rpc"); + describe("memory RPC endpoint", () => { + test("matches POST /api/v1/memory/rpc", () => { + const match = router.matchRoute("POST", "/api/v1/memory/rpc"); expect(match).not.toBeNull(); - expect(match?.route.pattern).toBe("/api/v1/accounts/rpc"); + expect(match?.route.pattern).toBe("/api/v1/memory/rpc"); expect(match?.params).toEqual({}); }); - test("does not match GET /api/v1/accounts/rpc", () => { - const match = router.matchRoute("GET", "/api/v1/accounts/rpc"); + test("does not match GET /api/v1/memory/rpc", () => { + const match = router.matchRoute("GET", "/api/v1/memory/rpc"); expect(match).toBeNull(); }); }); - describe("engine RPC endpoint", () => { - test("matches POST /api/v1/engine/rpc", () => { - const match = router.matchRoute("POST", "/api/v1/engine/rpc"); + describe("user RPC endpoint", () => { + test("matches POST /api/v1/user/rpc", () => { + const match = router.matchRoute("POST", "/api/v1/user/rpc"); expect(match).not.toBeNull(); - expect(match?.route.pattern).toBe("/api/v1/engine/rpc"); + expect(match?.route.pattern).toBe("/api/v1/user/rpc"); expect(match?.params).toEqual({}); }); - test("does not match GET /api/v1/engine/rpc", () => { - const match = router.matchRoute("GET", "/api/v1/engine/rpc"); + test("does not match GET /api/v1/user/rpc", () => { + const match = router.matchRoute("GET", "/api/v1/user/rpc"); expect(match).toBeNull(); }); }); @@ -214,7 +206,7 @@ describe("handleRequest", () => { const ctx = createMockContext(); const router = createRouter(ctx); - const request = new Request("http://localhost/api/v1/engine/rpc", { + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers: { "Content-Type": "application/json", @@ -242,7 +234,7 @@ describe("handleRequest", () => { const ctx = createMockContext(); const router = createRouter(ctx); - const request = new Request("http://localhost/api/v1/engine/rpc", { + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ diff --git a/packages/server/router.ts b/packages/server/router.ts index 2b88f27..a7230bf 100644 --- a/packages/server/router.ts +++ b/packages/server/router.ts @@ -10,20 +10,10 @@ import { } from "./handlers/auth"; import { healthHandler, readyHandler } from "./handlers/health"; import { versionHandler } from "./handlers/version"; -import { - authenticateAccounts, - authenticateEngine, -} from "./middleware/authenticate"; import { authenticateSpace } from "./middleware/authenticate-space"; import { authenticateUser } from "./middleware/authenticate-user"; import { checkClientVersion } from "./middleware/client-version"; -import { - accountsMethods, - createRpcHandler, - engineMethods, - memoryMethods, - userMethods, -} from "./rpc"; +import { createRpcHandler, memoryMethods, userMethods } from "./rpc"; import { notFound } from "./util/response"; /** @@ -129,9 +119,6 @@ export interface Router { */ export function createRouter(ctx: ServerContext): Router { const { - accountsDb, - accountsSql, - engineSql, db, auth, core, @@ -166,51 +153,6 @@ export function createRouter(ctx: ServerContext): Router { }; } - // Engine RPC: authenticate and provide db context - const engineRpcHandler = createRpcHandler(engineMethods, async (request) => { - const auth = await authenticateEngine(request, accountsDb, engineSql); - if (!auth.ok) { - return auth.error; - } - // TypeScript narrows auth.context to AuthContext after ok check - // We know it's EngineAuthContext since we called authenticateEngine - const ctx = auth.context; - if (ctx.type !== "engine") { - throw new Error("Unexpected auth context type"); - } - return { - db: ctx.db, - userId: ctx.userId, - apiKeyId: ctx.apiKeyId, - engine: ctx.engine, - embeddingConfig, - }; - }); - - // Accounts RPC: authenticate and provide identity context - const accountsRpcHandler = createRpcHandler( - accountsMethods, - async (request) => { - const result = await authenticateAccounts(request, auth); - if (!result.ok) { - return result.error; - } - // TypeScript narrows result.context to AuthContext after ok check - // We know it's AccountsAuthContext since we called authenticateAccounts - const authContext = result.context; - if (authContext.type !== "accounts") { - throw new Error("Unexpected auth context type"); - } - return { - db: accountsDb, - auth, - identity: authContext.identity, - engineSql, - serverVersion, - }; - }, - ); - // Memory RPC (new model): authenticate principal + space, provide space context const memoryRpcHandler = createRpcHandler(memoryMethods, async (request) => { const result = await authenticateSpace(request, { core, auth, db }); @@ -254,7 +196,7 @@ export function createRouter(ctx: ServerContext): Router { { method: "GET", pattern: "/ready", - handler: readyHandler(accountsSql, engineSql), + handler: readyHandler(db), }, // Version compatibility check (unauthenticated) @@ -304,20 +246,6 @@ export function createRouter(ctx: ServerContext): Router { handler: (req, params) => oauthCallbackHandler(req, params, authCtx), }, - // Accounts RPC - { - method: "POST", - pattern: "/api/v1/accounts/rpc", - handler: withClientVersionCheck(accountsRpcHandler), - }, - - // Engine RPC - { - method: "POST", - pattern: "/api/v1/engine/rpc", - handler: withClientVersionCheck(engineRpcHandler), - }, - // Memory RPC (new model: space data-plane + management) { method: "POST", diff --git a/packages/server/rpc/accounts/engine.integration.test.ts b/packages/server/rpc/accounts/engine.integration.test.ts deleted file mode 100644 index 9fd108e..0000000 --- a/packages/server/rpc/accounts/engine.integration.test.ts +++ /dev/null @@ -1,642 +0,0 @@ -/** - * Integration tests for engine provisioning via RPC. - * - * Tests that engine.create properly: - * 1. Creates engine record with correct language - * 2. Provisions schema in the engine database - * 3. Defaults language to "english" when not specified - */ - -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { - type AccountsDB, - createAccountsDB, - type Identity, -} from "@memory.build/accounts"; -import { TestDatabase as AccountsTestDatabase } from "@memory.build/accounts/migrate/test-utils"; -import { createEngineDB } from "@memory.build/engine"; -import { bootstrap } from "@memory.build/engine/migrate/bootstrap"; -import { TestDatabase as EngineTestDatabase } from "@memory.build/engine/migrate/test-utils"; -import { SQL } from "bun"; -import { SERVER_VERSION } from "../../../../version"; -import type { HandlerContext } from "../types"; -import { engineMethods } from "./engine"; -import type { AccountsRpcContext } from "./types"; - -// Test fixtures -let accountsTestDb: AccountsTestDatabase; -let engineTestDb: EngineTestDatabase; -let accountsDb: AccountsDB; -let engineSql: SQL; -let engineConnectionString: string; - -// Test data -let testOrgId: string; -let testIdentity: Identity; - -beforeAll(async () => { - // Set up accounts database - accountsTestDb = await AccountsTestDatabase.create(); - accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema); - - // Set up engine database - engineTestDb = new EngineTestDatabase(); - engineConnectionString = await engineTestDb.create(); - engineSql = new SQL(engineConnectionString); - - // Bootstrap the engine database (extensions, roles) - await bootstrap(engineSql); - - // Create test identity and org - testIdentity = await accountsDb.createIdentity({ - email: "engine-test@example.com", - name: "Engine Test User", - }); - - const org = await accountsDb.createOrg({ - name: "Engine Test Org", - }); - testOrgId = org.id; - - // Make identity an owner of the org - await accountsDb.addMember(org.id, testIdentity.id, "owner"); -}); - -afterAll(async () => { - await engineSql.close(); - await engineTestDb.drop(); - await accountsTestDb.dispose(); -}); - -/** - * Helper to create a context for engine methods. - */ -function createContext(identity: Identity): HandlerContext { - return { - request: new Request("http://localhost"), - db: accountsDb, - // engine handlers are legacy (AccountsDB) and never touch the auth store; - // a stub object satisfies the assertAccountsRpcContext guard. - auth: {} as unknown, - identity, - engineSql, - serverVersion: SERVER_VERSION, - } as unknown as AccountsRpcContext; -} - -/** - * Helper to check if a schema exists. - */ -async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.schemata - where schema_name = ${name} - ) as exists - `; - return row.exists; -} - -/** - * Helper to check if a table exists in a schema. - */ -async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = ${table} - ) as exists - `; - return row.exists; -} - -/** - * Helper to extract the text_config from the BM25 index definition. - * The config is embedded in the index during migration, not stored in a table. - */ -async function getBm25TextConfig( - sql: SQL, - schema: string, -): Promise { - try { - const [row] = await sql` - select pg_get_indexdef(indexrelid) as def - from pg_index i - join pg_class c on c.oid = i.indexrelid - join pg_namespace n on n.oid = c.relnamespace - where n.nspname = ${schema} - and c.relname = 'memory_content_bm25_idx' - `; - if (!row?.def) return null; - - // Extract text_config from: "... WITH (text_config=english, ..." (no quotes) - const match = row.def.match(/text_config=(\w+)/); - return match?.[1] ?? null; - } catch { - return null; - } -} - -// --------------------------------------------------------------------------- -// Engine Provisioning Tests -// --------------------------------------------------------------------------- - -describe("engine.create integration", () => { - test("creates engine record with default language (english)", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Default Language Engine" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify engine record - expect(result.id).toBeDefined(); - expect(result.slug).toMatch(/^[a-z0-9]{12}$/); - expect(result.language).toBe("english"); - - // Verify schema was provisioned - const schema = `me_${result.slug}`; - expect(await schemaExists(engineSql, schema)).toBe(true); - - // Verify core tables exist - expect(await tableExists(engineSql, schema, "memory")).toBe(true); - expect(await tableExists(engineSql, schema, "user")).toBe(true); - expect(await tableExists(engineSql, schema, "api_key")).toBe(true); - expect(await tableExists(engineSql, schema, "version")).toBe(true); - expect(await tableExists(engineSql, schema, "migration")).toBe(true); - - // Verify config has correct bm25_text_config - expect(await getBm25TextConfig(engineSql, schema)).toBe("english"); - }); - - test("creates engine record with custom language (german)", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "German Engine", language: "german" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify engine record - expect(result.id).toBeDefined(); - expect(result.language).toBe("german"); - - // Verify schema was provisioned - const schema = `me_${result.slug}`; - expect(await schemaExists(engineSql, schema)).toBe(true); - - // Verify config has correct bm25_text_config - expect(await getBm25TextConfig(engineSql, schema)).toBe("german"); - }); - - test("creates engine record with simple language (simple)", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Simple Engine", language: "simple" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify engine record - expect(result.language).toBe("simple"); - - // Verify schema was provisioned with simple text config - const schema = `me_${result.slug}`; - expect(await getBm25TextConfig(engineSql, schema)).toBe("simple"); - }); - - test("provisions schema with embedding_queue table", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Queue Test Engine" }, - context, - )) as { slug: string }; - - const schema = `me_${result.slug}`; - expect(await tableExists(engineSql, schema, "embedding_queue")).toBe(true); - }); - - test("rejects non-member creating engine", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - // Create a new identity that is NOT a member of the org - const outsider = await accountsDb.createIdentity({ - email: "outsider@example.com", - name: "Outsider", - }); - - const context = createContext(outsider); - - await expect( - handler({ orgId: testOrgId, name: "Unauthorized Engine" }, context), - ).rejects.toThrow("Only owners and admins can create engines"); - }); - - test("rejects member (non-admin) creating engine", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - // Create a new identity and add as regular member - const member = await accountsDb.createIdentity({ - email: "member@example.com", - name: "Regular Member", - }); - await accountsDb.addMember(testOrgId, member.id, "member"); - - const context = createContext(member); - - await expect( - handler({ orgId: testOrgId, name: "Member Engine" }, context), - ).rejects.toThrow("Only owners and admins can create engines"); - }); - - test("admin can create engine", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - // Create a new identity and add as admin - const admin = await accountsDb.createIdentity({ - email: "admin@example.com", - name: "Admin User", - }); - await accountsDb.addMember(testOrgId, admin.id, "admin"); - - const context = createContext(admin); - - const result = (await handler( - { orgId: testOrgId, name: "Admin Engine" }, - context, - )) as { id: string; slug: string }; - - expect(result.id).toBeDefined(); - const schema = `me_${result.slug}`; - expect(await schemaExists(engineSql, schema)).toBe(true); - }); - - test("engine record is persisted in accounts database", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Persisted Engine" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify we can fetch the engine from the accounts DB - const engine = await accountsDb.getEngine(result.id); - expect(engine).not.toBeNull(); - expect(engine?.name).toBe("Persisted Engine"); - expect(engine?.orgId).toBe(testOrgId); - expect(engine?.status).toBe("active"); - expect(engine?.language).toBe("english"); - }); - - test("engine can be retrieved by slug after creation", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Slug Lookup Engine" }, - context, - )) as { id: string; slug: string }; - - // Verify we can fetch the engine by slug - const engine = await accountsDb.getEngineBySlug(result.slug); - expect(engine).not.toBeNull(); - expect(engine?.id).toBe(result.id); - }); -}); - -// --------------------------------------------------------------------------- -// engine.update Tests -// --------------------------------------------------------------------------- - -describe("engine.update integration", () => { - function getCreateHandler() { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - return handler; - } - - function getUpdateHandler() { - const handler = engineMethods.get("engine.update")?.handler; - if (!handler) throw new Error("engine.update handler not found"); - return handler; - } - - test("owner can rename engine; slug is unchanged", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const context = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Rename Source" }, - context, - )) as { id: string; slug: string; updatedAt: string | null }; - - const result = (await update( - { id: created.id, name: "Renamed Engine" }, - context, - )) as { id: string; name: string; slug: string; updatedAt: string | null }; - - expect(result.id).toBe(created.id); - expect(result.name).toBe("Renamed Engine"); - expect(result.slug).toBe(created.slug); - expect(result.updatedAt).not.toBeNull(); - - // Verify the underlying schema name (which uses slug) is unaffected. - expect(await schemaExists(engineSql, `me_${created.slug}`)).toBe(true); - }); - - test("admin can rename engine", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const ownerCtx = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Admin Rename Source" }, - ownerCtx, - )) as { id: string }; - - const admin = await accountsDb.createIdentity({ - email: "rename-admin@example.com", - name: "Rename Admin", - }); - await accountsDb.addMember(testOrgId, admin.id, "admin"); - - const result = (await update( - { id: created.id, name: "Admin Renamed" }, - createContext(admin), - )) as { name: string }; - - expect(result.name).toBe("Admin Renamed"); - }); - - test("member (non-admin) cannot rename engine", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const ownerCtx = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Member Rename Source" }, - ownerCtx, - )) as { id: string }; - - const member = await accountsDb.createIdentity({ - email: "rename-member@example.com", - name: "Rename Member", - }); - await accountsDb.addMember(testOrgId, member.id, "member"); - - await expect( - update( - { id: created.id, name: "Forbidden Rename" }, - createContext(member), - ), - ).rejects.toThrow("Only owners and admins can update engines"); - }); - - test("non-member cannot rename engine", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const ownerCtx = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Outsider Rename Source" }, - ownerCtx, - )) as { id: string }; - - const outsider = await accountsDb.createIdentity({ - email: "rename-outsider@example.com", - name: "Rename Outsider", - }); - - await expect( - update( - { id: created.id, name: "Outsider Rename" }, - createContext(outsider), - ), - ).rejects.toThrow("Only owners and admins can update engines"); - }); - - test("rename to a sibling's name returns CONFLICT", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const context = createContext(testIdentity); - - await create({ orgId: testOrgId, name: "Conflict A" }, context); - const second = (await create( - { orgId: testOrgId, name: "Conflict B" }, - context, - )) as { id: string }; - - await expect( - update({ id: second.id, name: "Conflict A" }, context), - ).rejects.toThrow(/already exists in this organization/); - }); - - test("rename a non-existent engine returns NOT_FOUND", async () => { - const update = getUpdateHandler(); - const context = createContext(testIdentity); - - await expect( - update( - { id: "019d694f-79f6-7595-8faf-b70b01c11f98", name: "Nope" }, - context, - ), - ).rejects.toThrow(/Engine not found/); - }); -}); - -// --------------------------------------------------------------------------- -// engine.setupAccess Tests -// --------------------------------------------------------------------------- - -describe("engine.setupAccess integration", () => { - // Create a dedicated engine for setupAccess tests - let setupAccessEngineId: string; - let setupAccessEngineSlug: string; - - beforeAll(async () => { - const createHandler = engineMethods.get("engine.create")?.handler; - if (!createHandler) throw new Error("engine.create handler not found"); - - const result = (await createHandler( - { orgId: testOrgId, name: "SetupAccess Test Engine" }, - createContext(testIdentity), - )) as { id: string; slug: string }; - - setupAccessEngineId = result.id; - setupAccessEngineSlug = result.slug; - }); - - function getHandler() { - const handler = engineMethods.get("engine.setupAccess")?.handler; - if (!handler) throw new Error("engine.setupAccess handler not found"); - return handler; - } - - test("owner gets superuser + createrole user and API key", async () => { - const handler = getHandler(); - const context = createContext(testIdentity); - - const result = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { - rawKey: string; - engineSlug: string; - userId: string; - engineName: string; - orgName: string; - }; - - expect(result.rawKey).toBeDefined(); - expect(result.rawKey.length).toBeGreaterThan(0); - expect(result.engineSlug).toBe(setupAccessEngineSlug); - expect(result.userId).toBeDefined(); - expect(result.engineName).toBe("SetupAccess Test Engine"); - expect(result.orgName).toBe("Engine Test Org"); - - // Verify the engine user has superuser privileges - const engineDb = createEngineDB(engineSql, `me_${setupAccessEngineSlug}`); - const user = await engineDb.getUser(result.userId); - expect(user).not.toBeNull(); - expect(user?.superuser).toBe(true); - expect(user?.createrole).toBe(true); - expect(user?.identityId).toBe(testIdentity.id); - }); - - test("admin gets superuser + createrole user and API key", async () => { - const handler = getHandler(); - - const admin = await accountsDb.createIdentity({ - email: "setup-admin@example.com", - name: "Setup Admin", - }); - await accountsDb.addMember(testOrgId, admin.id, "admin"); - - const context = createContext(admin); - const result = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - expect(result.rawKey).toBeDefined(); - - const engineDb = createEngineDB(engineSql, `me_${setupAccessEngineSlug}`); - const user = await engineDb.getUser(result.userId); - expect(user?.superuser).toBe(true); - expect(user?.createrole).toBe(true); - }); - - test("member gets vanilla user (no superuser) and API key", async () => { - const handler = getHandler(); - - const member = await accountsDb.createIdentity({ - email: "setup-member@example.com", - name: "Setup Member", - }); - await accountsDb.addMember(testOrgId, member.id, "member"); - - const context = createContext(member); - const result = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - expect(result.rawKey).toBeDefined(); - - const engineDb = createEngineDB(engineSql, `me_${setupAccessEngineSlug}`); - const user = await engineDb.getUser(result.userId); - expect(user?.superuser).toBe(false); - expect(user?.createrole).toBe(false); - }); - - test("non-member is forbidden", async () => { - const handler = getHandler(); - - const outsider = await accountsDb.createIdentity({ - email: "setup-outsider@example.com", - name: "Setup Outsider", - }); - - const context = createContext(outsider); - - await expect( - handler({ engineId: setupAccessEngineId }, context), - ).rejects.toThrow("Not a member of the organization"); - }); - - test("engine not found returns error", async () => { - const handler = getHandler(); - const context = createContext(testIdentity); - - await expect( - handler({ engineId: "019d694f-79f6-7595-8faf-b70b01c11f98" }, context), - ).rejects.toThrow("Engine not found"); - }); - - test("idempotent: second call reuses user, creates new API key", async () => { - const handler = getHandler(); - - const idempotentUser = await accountsDb.createIdentity({ - email: "setup-idempotent@example.com", - name: "Idempotent User", - }); - await accountsDb.addMember(testOrgId, idempotentUser.id, "owner"); - - const context = createContext(idempotentUser); - - // First call - const result1 = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - // Second call - const result2 = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - // Same user, different API keys - expect(result2.userId).toBe(result1.userId); - expect(result2.rawKey).not.toBe(result1.rawKey); - }); - - test("custom API key name is used", async () => { - const handler = getHandler(); - - const namedKeyUser = await accountsDb.createIdentity({ - email: "setup-named@example.com", - name: "Named Key User", - }); - await accountsDb.addMember(testOrgId, namedKeyUser.id, "owner"); - - const context = createContext(namedKeyUser); - const result = (await handler( - { engineId: setupAccessEngineId, apiKeyName: "my-custom-key" }, - context, - )) as { rawKey: string }; - - expect(result.rawKey).toBeDefined(); - }); -}); diff --git a/packages/server/rpc/accounts/engine.ts b/packages/server/rpc/accounts/engine.ts deleted file mode 100644 index 9fe4e29..0000000 --- a/packages/server/rpc/accounts/engine.ts +++ /dev/null @@ -1,432 +0,0 @@ -/** - * Accounts RPC engine methods. - * - * Implements: - * - engine.create: Create a new engine for an organization - * - engine.list: List engines for an organization - * - engine.get: Get engine by ID - * - engine.update: Update engine name/status - * - engine.delete: Delete an engine (mark deleted + drop schema) - * - engine.setupAccess: Bootstrap engine access for a session-authenticated identity - */ -import type { Engine } from "@memory.build/accounts"; -import { - createEngineDB, - type EngineConfig, - provisionEngine, -} from "@memory.build/engine"; -import { setLocalEngineTimeouts } from "@memory.build/engine/ops/_tx"; -import type { - EngineCreateParams, - EngineDeleteParams, - EngineGetParams, - EngineListParams, - EngineResponse, - EngineSetupAccessParams, - EngineSetupAccessResult, - EngineUpdateParams, -} from "@memory.build/protocol/accounts/engine"; -import { - engineCreateParams, - engineDeleteParams, - engineGetParams, - engineListParams, - engineSetupAccessParams, - engineUpdateParams, -} from "@memory.build/protocol/accounts/engine"; -import { SQL } from "bun"; -import { embeddingConstants } from "../../config"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an Engine to a serializable response. - */ -function toEngineResponse(engine: Engine): EngineResponse { - return { - id: engine.id, - orgId: engine.orgId, - slug: engine.slug, - name: engine.name, - shardId: engine.shardId, - status: engine.status, - language: engine.language, - createdAt: engine.createdAt.toISOString(), - updatedAt: engine.updatedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * engine.create - Create a new engine for an organization. - * Requires owner or admin role. - */ -async function engineCreate( - params: EngineCreateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity, engineSql, serverVersion } = - context as AccountsRpcContext; - - // Check if caller has admin or owner role - const member = await db.getMember(params.orgId, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can create engines", - ); - } - - // Create the engine record in accounts DB - const engine = await db.createEngine({ - orgId: params.orgId, - name: params.name, - language: params.language ?? "english", - }); - - // Provision the engine schema in the engine DB - const engineConfig: EngineConfig = { - embedding_dimensions: embeddingConstants.dimensions, - bm25_text_config: engine.language, - }; - - try { - await provisionEngine( - engineSql, - engine.slug, - engineConfig, - serverVersion, - engine.shardId, - ); - } catch (err) { - // Attempt to clean up partially-created schema - const schema = `me_${engine.slug}`; - try { - await engineSql.begin(async (tx) => { - await tx.unsafe(`set local pgdog.shard to ${engine.shardId}`); - await setLocalEngineTimeouts(tx); - await tx.unsafe(`drop schema if exists ${schema} cascade`); - }); - } catch { - // Log but don't mask original error - } - // Mark engine as deleted in accounts DB - await db.updateEngine(engine.id, { status: "deleted" }); - throw new AppError( - "INTERNAL_ERROR", - `Failed to provision engine schema: ${err instanceof Error ? err.message : String(err)}`, - ); - } - - return toEngineResponse(engine); -} - -/** - * engine.list - List engines for an organization. - * Requires membership in the org. - */ -async function engineList( - params: EngineListParams, - context: HandlerContext, -): Promise<{ engines: EngineResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const member = await db.getMember(params.orgId, identity.id); - if (!member) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const engines = await db.listEnginesByOrg(params.orgId); - return { engines: engines.map(toEngineResponse) }; -} - -/** - * engine.get - Get engine by ID. - * Requires membership in the org that owns the engine. - */ -async function engineGet( - params: EngineGetParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - const engine = await db.getEngine(params.id); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - // Check if caller is a member of the org - const member = await db.getMember(engine.orgId, identity.id); - if (!member) { - throw new AppError( - "FORBIDDEN", - "Not a member of the organization that owns this engine", - ); - } - - return toEngineResponse(engine); -} - -/** - * engine.update - Update engine name/status. - * Requires owner or admin role. - */ -async function engineUpdate( - params: EngineUpdateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - const engine = await db.getEngine(params.id); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - // Check if caller has admin or owner role - const member = await db.getMember(engine.orgId, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can update engines", - ); - } - - let updated: boolean; - try { - updated = await db.updateEngine(params.id, { - name: params.name, - status: params.status, - }); - } catch (err) { - // Translate the unique-name violation on (org_id, name) into a - // friendly CONFLICT error so the CLI can show a clean message. - if ( - err instanceof SQL.PostgresError && - err.errno === "23505" && - typeof err.constraint === "string" && - err.constraint.includes("name") - ) { - throw new AppError( - "CONFLICT", - `An engine named '${params.name}' already exists in this organization`, - ); - } - throw err; - } - - if (!updated) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - const updatedEngine = await db.getEngine(params.id); - if (!updatedEngine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - return toEngineResponse(updatedEngine); -} - -/** - * engine.setupAccess - Bootstrap engine access for a session-authenticated identity. - * - * Find-or-creates an engine user for the caller's identity, then creates an API key. - * Any org member can call this. Privilege level maps from org role: - * - owner/admin → superuser + createrole - * - member → vanilla user - */ -async function engineSetupAccess( - params: EngineSetupAccessParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity, engineSql } = context as AccountsRpcContext; - - // Look up the engine - const engine = await db.getEngine(params.engineId); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.engineId}`); - } - if (engine.status !== "active") { - throw new AppError( - "VALIDATION_ERROR", - `Engine is not active: ${engine.status}`, - ); - } - - // Look up the org - const org = await db.getOrg(engine.orgId); - if (!org) { - throw new AppError("NOT_FOUND", `Organization not found: ${engine.orgId}`); - } - - // Check caller's membership - const member = await db.getMember(engine.orgId, identity.id); - if (!member) { - throw new AppError( - "FORBIDDEN", - "Not a member of the organization that owns this engine", - ); - } - - // Create an EngineDB for this engine's schema - const schema = `me_${engine.slug}`; - const engineDb = createEngineDB(engineSql, schema, { shard: engine.shardId }); - - // Find or create a user for this identity - let user = await engineDb.getUserByIdentity(identity.id); - if (!user) { - const isSuperuser = member.role === "owner" || member.role === "admin"; - user = await engineDb.createUser({ - name: slugifyUserName(identity.name || identity.email), - identityId: identity.id, - canLogin: true, - superuser: isSuperuser, - createrole: isSuperuser, - }); - } - - // Create an API key for the user - const apiKeyName = - params.apiKeyName ?? `cli-${new Date().toISOString().slice(0, 10)}`; - const { rawKey } = await engineDb.createApiKey({ - userId: user.id, - name: apiKeyName, - }); - - return { - rawKey, - engineSlug: engine.slug, - userId: user.id, - engineName: engine.name, - orgName: org.name, - }; -} - -/** - * Derive a shell-friendly engine user name from a display name or email. - * Lowercases, replaces whitespace runs with hyphens, strips non-alphanumeric - * characters (except hyphens), and trims leading/trailing hyphens. - */ -function slugifyUserName(name: string): string { - return name - .toLowerCase() - .replace(/\s+/g, "-") - .replace(/[^a-z0-9-]/g, "") - .replace(/-{2,}/g, "-") - .replace(/^-|-$/g, ""); -} - -/** - * engine.delete - Delete an engine permanently. - * - * Three-step sequence (each step is idempotent so a failed prior attempt - * can be retried to completion): - * 1. Mark the engine row status='deleted' so middleware rejects new - * API-key auth immediately, even if subsequent steps stall. - * 2. Drop the engine schema (uses `drop schema if exists`). - * 3. Hard-delete the engine row from the accounts DB so it no longer - * blocks `org.delete` and disappears from listings. - * - * Requires owner role on the org. - */ -async function engineDelete( - params: EngineDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertAccountsRpcContext(context); - const { db, identity, engineSql } = context as AccountsRpcContext; - - const engine = await db.getEngine(params.id); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - // Only org owners can delete engines - const member = await db.getMember(engine.orgId, identity.id); - if (!member || member.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can delete engines"); - } - - // Step 1: Mark as deleted (idempotent — bumping updated_at on a row that - // is already 'deleted' is harmless and lets us proceed to retry steps - // 2 and 3 if a previous attempt failed mid-way). - if (engine.status !== "deleted") { - await db.updateEngine(engine.id, { status: "deleted" }); - } - - // Step 2: Drop the engine schema. `drop schema if exists` is idempotent. - const schema = `me_${engine.slug}`; - await dropSchemaWithRetry(engineSql, schema, { - retries: 3, - delayMs: 2000, - shardId: engine.shardId, - }); - - // Step 3: Hard-delete the row so it no longer blocks org deletion or - // appears in engine listings. Must come after the schema drop — if we - // delete the row first and the schema drop then fails, the schema is - // orphaned with no metadata to find it. - await db.deleteEngine(engine.id); - - return { deleted: true }; -} - -/** - * Drop an engine schema with retry logic. - * Sets a lock_timeout to avoid blocking indefinitely if the embedding worker - * or in-flight requests hold locks, then retries on timeout. - * Sets pgdog.shard for correct shard routing. - */ -async function dropSchemaWithRetry( - sql: import("bun").SQL, - schema: string, - opts: { retries: number; delayMs: number; shardId: number }, -): Promise { - for (let attempt = 1; attempt <= opts.retries; attempt++) { - try { - await sql.begin(async (tx) => { - await tx.unsafe(`set local pgdog.shard to ${opts.shardId}`); - await setLocalEngineTimeouts(tx); - await tx.unsafe(`set local lock_timeout = '5s'`); - await tx.unsafe(`drop schema if exists ${schema} cascade`); - }); - return; - } catch (err) { - const isLockTimeout = - err instanceof Error && err.message?.includes("lock timeout"); - if (!isLockTimeout || attempt === opts.retries) { - throw new AppError( - "INTERNAL_ERROR", - `Failed to drop engine schema ${schema}: ${err instanceof Error ? err.message : String(err)}`, - ); - } - await new Promise((resolve) => setTimeout(resolve, opts.delayMs)); - } - } -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the engine methods registry. - */ -export const engineMethods = buildRegistry() - .register("engine.create", engineCreateParams, engineCreate) - .register("engine.list", engineListParams, engineList) - .register("engine.get", engineGetParams, engineGet) - .register("engine.update", engineUpdateParams, engineUpdate) - .register("engine.delete", engineDeleteParams, engineDelete) - .register("engine.setupAccess", engineSetupAccessParams, engineSetupAccess) - .build(); diff --git a/packages/server/rpc/accounts/index.ts b/packages/server/rpc/accounts/index.ts deleted file mode 100644 index d06aca2..0000000 --- a/packages/server/rpc/accounts/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { buildRegistry } from "../registry"; -import { engineMethods } from "./engine"; -import { invitationMethods } from "./invitation"; -import { meMethods } from "./me"; -import { orgMethods } from "./org"; -import { orgMemberMethods } from "./org-member"; -import { sessionMethods } from "./session"; - -/** - * Accounts RPC method registry. - * - * Identity methods: - * - me.get - * - * Session methods: - * - session.revoke - * - * Organization methods: - * - org.create, org.list, org.get, org.update, org.delete - * - * Organization member methods: - * - org.member.list, org.member.add, org.member.remove, org.member.updateRole - * - * Engine methods: - * - engine.create, engine.list, engine.get, engine.update - * - * Invitation methods: - * - invitation.create, invitation.list, invitation.revoke, invitation.accept - */ -export const accountsMethods = buildRegistry() - .merge(meMethods) - .merge(sessionMethods) - .merge(orgMethods) - .merge(orgMemberMethods) - .merge(engineMethods) - .merge(invitationMethods) - .build(); - -// Re-export types for consumers -export type { AccountsRpcContext } from "./types"; -export { assertAccountsRpcContext, isAccountsRpcContext } from "./types"; diff --git a/packages/server/rpc/accounts/invitation.ts b/packages/server/rpc/accounts/invitation.ts deleted file mode 100644 index 8b5a3e6..0000000 --- a/packages/server/rpc/accounts/invitation.ts +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Accounts RPC invitation methods. - * - * Implements: - * - invitation.create: Create an invitation to join an organization - * - invitation.list: List pending invitations for an organization - * - invitation.revoke: Revoke a pending invitation - * - invitation.accept: Accept an invitation (adds caller to org) - */ -import type { Invitation } from "@memory.build/accounts"; -import type { - InvitationAcceptParams, - InvitationCreateParams, - InvitationCreateResult, - InvitationListParams, - InvitationResponse, - InvitationRevokeParams, -} from "@memory.build/protocol/accounts/invitation"; -import { - invitationAcceptParams, - invitationCreateParams, - invitationListParams, - invitationRevokeParams, -} from "@memory.build/protocol/accounts/invitation"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an Invitation to a serializable response. - */ -function toInvitationResponse(invitation: Invitation): InvitationResponse { - return { - id: invitation.id, - orgId: invitation.orgId, - email: invitation.email, - role: invitation.role, - invitedBy: invitation.invitedBy, - expiresAt: invitation.expiresAt.toISOString(), - acceptedAt: invitation.acceptedAt?.toISOString() ?? null, - createdAt: invitation.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * invitation.create - Create an invitation to join an organization. - * Requires owner or admin role. - * Returns the invitation with the raw token (only shown once). - */ -async function invitationCreate( - params: InvitationCreateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const member = await db.getMember(params.orgId, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can create invitations", - ); - } - - // Only owners can invite other owners - if (params.role === "owner" && member.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can invite other owners"); - } - - const result = await db.createInvitation({ - orgId: params.orgId, - email: params.email, - role: params.role, - invitedBy: identity.id, - expiresInDays: params.expiresInDays, - }); - - return { - ...toInvitationResponse(result.invitation), - token: result.rawToken, - }; -} - -/** - * invitation.list - List pending invitations for an organization. - * Requires membership in the org. - */ -async function invitationList( - params: InvitationListParams, - context: HandlerContext, -): Promise<{ invitations: InvitationResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const member = await db.getMember(params.orgId, identity.id); - if (!member) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const invitations = await db.listPendingInvitations(params.orgId); - return { invitations: invitations.map(toInvitationResponse) }; -} - -/** - * invitation.revoke - Revoke a pending invitation. - * Requires owner or admin role. - */ -async function invitationRevoke( - params: InvitationRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // We need to find the invitation first to check org membership - // Since we don't have getInvitation by ID, we'll need to verify via different means - // For now, we'll revoke and let the database handle the not found case - // The authorization check happens via listing invitations for orgs the user is admin of - - // Get all orgs the caller is admin/owner of - const orgs = await db.listOrgsByIdentity(identity.id); - - // For each org, check if the invitation belongs to it and caller has permission - for (const org of orgs) { - const member = await db.getMember(org.id, identity.id); - if (member && (member.role === "owner" || member.role === "admin")) { - const invitations = await db.listPendingInvitations(org.id); - const invitation = invitations.find((inv) => inv.id === params.id); - if (invitation) { - const revoked = await db.revokeInvitation(params.id); - return { revoked }; - } - } - } - - throw new AppError("NOT_FOUND", `Invitation not found: ${params.id}`); -} - -/** - * invitation.accept - Accept an invitation (adds caller to org). - * The caller's email must match the invitation email. - */ -async function invitationAccept( - params: InvitationAcceptParams, - context: HandlerContext, -): Promise<{ accepted: boolean; orgId: string }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Find the invitation by token - const invitation = await db.getInvitationByToken(params.token); - if (!invitation) { - throw new AppError("NOT_FOUND", "Invalid or expired invitation token"); - } - - // Check email matches (identity already available from auth - no DB lookup needed) - if (invitation.email.toLowerCase() !== identity.email.toLowerCase()) { - throw new AppError( - "FORBIDDEN", - "Invitation is for a different email address", - ); - } - - // Accept the invitation and add member in a transaction - await db.withTransaction(async (txDb) => { - const accepted = await txDb.acceptInvitation(invitation.id); - if (!accepted) { - throw new AppError("CONFLICT", "Invitation has already been accepted"); - } - - // Add the user as a member with the invited role - await txDb.addMember(invitation.orgId, identity.id, invitation.role); - }); - - return { accepted: true, orgId: invitation.orgId }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the invitation methods registry. - */ -export const invitationMethods = buildRegistry() - .register("invitation.create", invitationCreateParams, invitationCreate) - .register("invitation.list", invitationListParams, invitationList) - .register("invitation.revoke", invitationRevokeParams, invitationRevoke) - .register("invitation.accept", invitationAcceptParams, invitationAccept) - .build(); diff --git a/packages/server/rpc/accounts/me.test.ts b/packages/server/rpc/accounts/me.test.ts deleted file mode 100644 index 3f9e637..0000000 --- a/packages/server/rpc/accounts/me.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Unit tests for identity/me RPC handlers. - * - * Uses a mocked AuthStore to test handler logic in isolation. - */ -import { describe, expect, mock, test } from "bun:test"; -import type { User } from "@memory.build/auth"; -import type { HandlerContext } from "../types"; -import { meMethods } from "./me"; - -const authUser: User = { - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "alice@example.com", - name: "Alice", - emailVerified: true, - image: null, - createdAt: new Date("2026-01-15T00:00:00.000Z"), - updatedAt: null, -}; - -function createMockContext( - authOverrides: Record = {}, -): HandlerContext { - return { - request: new Request("http://localhost"), - // Legacy AccountsDB stub — present only to satisfy the context guard. - db: {}, - auth: { - getUser: mock(() => Promise.resolve(authUser)), - getUserByEmail: mock(() => Promise.resolve(null)), - ...authOverrides, - }, - identity: { - id: authUser.id, - email: authUser.email, - name: authUser.name, - }, - engineSql: mock(() => {}) as unknown, - serverVersion: "0.1.1", - } as unknown as HandlerContext; -} - -// ============================================================================= -// me.get -// ============================================================================= - -describe("me.get", () => { - test("returns the authenticated user", async () => { - const handler = meMethods.get("me.get")?.handler; - if (!handler) throw new Error("me.get handler not found"); - - const context = createMockContext(); - const result = (await handler({}, context)) as { - id: string; - email: string; - name: string; - }; - - expect(result.id).toBe(authUser.id); - expect(result.email).toBe("alice@example.com"); - expect(result.name).toBe("Alice"); - }); - - test("looks up the user by the session identity id", async () => { - const handler = meMethods.get("me.get")?.handler; - if (!handler) throw new Error("me.get handler not found"); - - const getUser = mock(() => Promise.resolve(authUser)); - const context = createMockContext({ getUser }); - - await handler({}, context); - - expect(getUser).toHaveBeenCalledWith(authUser.id); - }); -}); - -// ============================================================================= -// identity.getByEmail -// ============================================================================= - -describe("identity.getByEmail", () => { - test("returns identity when found", async () => { - const handler = meMethods.get("identity.getByEmail")?.handler; - if (!handler) throw new Error("identity.getByEmail handler not found"); - - const bob: User = { - id: "019d694f-79f6-7595-8faf-b70b01c11f99", - email: "bob@example.com", - name: "Bob", - emailVerified: true, - image: null, - createdAt: new Date("2026-01-15T00:00:00.000Z"), - updatedAt: null, - }; - - const context = createMockContext({ - getUserByEmail: mock(() => Promise.resolve(bob)), - }); - - const result = (await handler({ email: "bob@example.com" }, context)) as { - identity: { id: string; email: string; name: string } | null; - }; - - expect(result.identity).not.toBeNull(); - expect(result.identity!.id).toBe("019d694f-79f6-7595-8faf-b70b01c11f99"); - expect(result.identity!.email).toBe("bob@example.com"); - expect(result.identity!.name).toBe("Bob"); - }); - - test("returns null identity when not found", async () => { - const handler = meMethods.get("identity.getByEmail")?.handler; - if (!handler) throw new Error("identity.getByEmail handler not found"); - - const context = createMockContext({ - getUserByEmail: mock(() => Promise.resolve(null)), - }); - - const result = (await handler( - { email: "nobody@example.com" }, - context, - )) as { identity: null }; - - expect(result.identity).toBeNull(); - }); - - test("passes email to the auth-store lookup", async () => { - const handler = meMethods.get("identity.getByEmail")?.handler; - if (!handler) throw new Error("identity.getByEmail handler not found"); - - const getUserByEmail = mock(() => Promise.resolve(null)); - const context = createMockContext({ getUserByEmail }); - - await handler({ email: "test@example.com" }, context); - - expect(getUserByEmail).toHaveBeenCalledWith("test@example.com"); - }); -}); diff --git a/packages/server/rpc/accounts/me.ts b/packages/server/rpc/accounts/me.ts deleted file mode 100644 index f0539ab..0000000 --- a/packages/server/rpc/accounts/me.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Accounts RPC me/identity methods (auth-schema backed). - */ -import type { User } from "@memory.build/auth"; -import type { - IdentityGetByEmailParams, - IdentityGetByEmailResult, - IdentityResponse, - MeGetParams, -} from "@memory.build/protocol/accounts/identity"; -import { - identityGetByEmailParams, - meGetParams, -} from "@memory.build/protocol/accounts/identity"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -function toIdentityResponse(user: User): IdentityResponse { - return { - id: user.id, - email: user.email, - name: user.name, - createdAt: user.createdAt.toISOString(), - updatedAt: user.updatedAt?.toISOString() ?? null, - }; -} - -/** me.get — the current authenticated user. */ -async function meGet( - _params: MeGetParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { auth, identity } = context as AccountsRpcContext; - const user = await auth.getUser(identity.id); - if (!user) { - throw new Error("Authenticated user not found"); - } - return toIdentityResponse(user); -} - -/** identity.getByEmail — look up a user by email. */ -async function identityGetByEmail( - params: IdentityGetByEmailParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { auth } = context as AccountsRpcContext; - const user = await auth.getUserByEmail(params.email); - return { identity: user ? toIdentityResponse(user) : null }; -} - -export const meMethods = buildRegistry() - .register("me.get", meGetParams, meGet) - .register("identity.getByEmail", identityGetByEmailParams, identityGetByEmail) - .build(); diff --git a/packages/server/rpc/accounts/org-member.ts b/packages/server/rpc/accounts/org-member.ts deleted file mode 100644 index e397d2a..0000000 --- a/packages/server/rpc/accounts/org-member.ts +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Accounts RPC org member methods. - * - * Implements: - * - org.member.list: List members of an organization - * - org.member.add: Add a member to an organization - * - org.member.remove: Remove a member from an organization - * - org.member.updateRole: Update a member's role - */ -import { AccountsError, type OrgMember } from "@memory.build/accounts"; -import type { - OrgMemberAddParams, - OrgMemberListParams, - OrgMemberRemoveParams, - OrgMemberResponse, - OrgMemberUpdateRoleParams, -} from "@memory.build/protocol/accounts/org-member"; -import { - orgMemberAddParams, - orgMemberListParams, - orgMemberRemoveParams, - orgMemberUpdateRoleParams, -} from "@memory.build/protocol/accounts/org-member"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an OrgMember to a serializable response. - */ -function toOrgMemberResponse(member: OrgMember): OrgMemberResponse { - return { - orgId: member.orgId, - identityId: member.identityId, - role: member.role, - name: member.name, - email: member.email, - createdAt: member.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * org.member.list - List members of an organization. - * Requires membership in the org. - */ -async function orgMemberList( - params: OrgMemberListParams, - context: HandlerContext, -): Promise<{ members: OrgMemberResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const callerMember = await db.getMember(params.orgId, identity.id); - if (!callerMember) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const members = await db.listMembers(params.orgId); - return { members: members.map(toOrgMemberResponse) }; -} - -/** - * org.member.add - Add a member to an organization. - * Requires owner or admin role. - */ -async function orgMemberAdd( - params: OrgMemberAddParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const callerMember = await db.getMember(params.orgId, identity.id); - if ( - !callerMember || - (callerMember.role !== "owner" && callerMember.role !== "admin") - ) { - throw new AppError("FORBIDDEN", "Only owners and admins can add members"); - } - - // Only owners can add other owners - if (params.role === "owner" && callerMember.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can add other owners"); - } - - const member = await db.addMember( - params.orgId, - params.identityId, - params.role, - ); - return toOrgMemberResponse(member); -} - -/** - * org.member.remove - Remove a member from an organization. - * Requires owner or admin role (admins cannot remove owners). - */ -async function orgMemberRemove( - params: OrgMemberRemoveParams, - context: HandlerContext, -): Promise<{ removed: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const callerMember = await db.getMember(params.orgId, identity.id); - if ( - !callerMember || - (callerMember.role !== "owner" && callerMember.role !== "admin") - ) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can remove members", - ); - } - - // Check target member's role - const targetMember = await db.getMember(params.orgId, params.identityId); - if (!targetMember) { - throw new AppError("NOT_FOUND", "Member not found"); - } - - // Admins cannot remove owners - if (targetMember.role === "owner" && callerMember.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can remove other owners"); - } - - try { - const removed = await db.removeMember(params.orgId, params.identityId); - return { removed }; - } catch (err) { - if (err instanceof AccountsError && err.code === "ORG_MUST_HAVE_OWNER") { - throw new AppError( - "CONFLICT", - "Cannot remove the last owner from an organization", - ); - } - throw err; - } -} - -/** - * org.member.updateRole - Update a member's role. - * Requires owner or admin role (admins cannot promote to owner or demote owners). - */ -async function orgMemberUpdateRole( - params: OrgMemberUpdateRoleParams, - context: HandlerContext, -): Promise<{ updated: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const callerMember = await db.getMember(params.orgId, identity.id); - if ( - !callerMember || - (callerMember.role !== "owner" && callerMember.role !== "admin") - ) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can update member roles", - ); - } - - // Check target member's current role - const targetMember = await db.getMember(params.orgId, params.identityId); - if (!targetMember) { - throw new AppError("NOT_FOUND", "Member not found"); - } - - // Only owners can: - // - Promote to owner - // - Demote from owner - if ( - (params.role === "owner" || targetMember.role === "owner") && - callerMember.role !== "owner" - ) { - throw new AppError( - "FORBIDDEN", - "Only owners can promote to or demote from owner role", - ); - } - - try { - const updated = await db.updateRole( - params.orgId, - params.identityId, - params.role, - ); - return { updated }; - } catch (err) { - if (err instanceof AccountsError && err.code === "ORG_MUST_HAVE_OWNER") { - throw new AppError( - "CONFLICT", - "Cannot remove the last owner from an organization", - ); - } - throw err; - } -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the org member methods registry. - */ -export const orgMemberMethods = buildRegistry() - .register("org.member.list", orgMemberListParams, orgMemberList) - .register("org.member.add", orgMemberAddParams, orgMemberAdd) - .register("org.member.remove", orgMemberRemoveParams, orgMemberRemove) - .register( - "org.member.updateRole", - orgMemberUpdateRoleParams, - orgMemberUpdateRole, - ) - .build(); diff --git a/packages/server/rpc/accounts/org.integration.test.ts b/packages/server/rpc/accounts/org.integration.test.ts deleted file mode 100644 index eb2d00f..0000000 --- a/packages/server/rpc/accounts/org.integration.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Integration tests for org RPC handlers. - * - * Currently focused on org.update (rename). Other org methods are exercised - * indirectly via engine.integration.test.ts and CLI smoke tests. - */ - -import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; -import { - type AccountsDB, - createAccountsDB, - type Identity, -} from "@memory.build/accounts"; -import { TestDatabase as AccountsTestDatabase } from "@memory.build/accounts/migrate/test-utils"; -import { SERVER_VERSION } from "../../../../version"; -import type { HandlerContext } from "../types"; -import { orgMethods } from "./org"; -import type { AccountsRpcContext } from "./types"; - -let accountsTestDb: AccountsTestDatabase; -let accountsDb: AccountsDB; -let testIdentity: Identity; - -beforeAll(async () => { - accountsTestDb = await AccountsTestDatabase.create(); - accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema); - - testIdentity = await accountsDb.createIdentity({ - email: "org-rpc-test@example.com", - name: "Org RPC Test User", - }); -}); - -afterAll(async () => { - await accountsTestDb.dispose(); -}); - -function createContext(identity: Identity): HandlerContext { - return { - request: new Request("http://localhost"), - db: accountsDb, - // org handlers are legacy (AccountsDB) and never touch the auth store; - // a stub object satisfies the assertAccountsRpcContext guard. - auth: {} as unknown, - identity, - // org.update never touches engineSql; a stub that satisfies the - // assertAccountsRpcContext type guard (typeof === "function") is enough. - engineSql: mock(() => {}) as unknown, - serverVersion: SERVER_VERSION, - } as unknown as AccountsRpcContext; -} - -describe("org.update integration", () => { - function getUpdateHandler() { - const handler = orgMethods.get("org.update")?.handler; - if (!handler) throw new Error("org.update handler not found"); - return handler; - } - - test("owner can rename org; slug is unchanged", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Original Org Name" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const result = (await update( - { id: org.id, name: "Renamed Org" }, - createContext(testIdentity), - )) as { id: string; name: string; slug: string; updatedAt: string | null }; - - expect(result.id).toBe(org.id); - expect(result.name).toBe("Renamed Org"); - expect(result.slug).toBe(org.slug); - expect(result.updatedAt).not.toBeNull(); - }); - - test("admin can rename org", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Admin Rename" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const admin = await accountsDb.createIdentity({ - email: "org-admin@example.com", - name: "Org Admin", - }); - await accountsDb.addMember(org.id, admin.id, "admin"); - - const result = (await update( - { id: org.id, name: "Renamed By Admin" }, - createContext(admin), - )) as { name: string }; - - expect(result.name).toBe("Renamed By Admin"); - }); - - test("member (non-admin) cannot rename org", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Member Rename" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const member = await accountsDb.createIdentity({ - email: "org-member@example.com", - name: "Org Member", - }); - await accountsDb.addMember(org.id, member.id, "member"); - - await expect( - update({ id: org.id, name: "Forbidden" }, createContext(member)), - ).rejects.toThrow("Only owners and admins can update the organization"); - }); - - test("non-member cannot rename org", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Outsider Rename" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const outsider = await accountsDb.createIdentity({ - email: "org-outsider@example.com", - name: "Org Outsider", - }); - - await expect( - update({ id: org.id, name: "Forbidden" }, createContext(outsider)), - ).rejects.toThrow("Only owners and admins can update the organization"); - }); - - test("two orgs can share the same name (no unique constraint)", async () => { - const update = getUpdateHandler(); - - const orgA = await accountsDb.createOrg({ name: "Shared Name Source" }); - await accountsDb.addMember(orgA.id, testIdentity.id, "owner"); - const orgB = await accountsDb.createOrg({ name: "Other Org" }); - await accountsDb.addMember(orgB.id, testIdentity.id, "owner"); - - const result = (await update( - { id: orgB.id, name: "Shared Name Source" }, - createContext(testIdentity), - )) as { name: string }; - - expect(result.name).toBe("Shared Name Source"); - const fetched = await accountsDb.getOrg(orgA.id); - expect(fetched?.name).toBe("Shared Name Source"); - }); - - test("renaming a non-existent org returns FORBIDDEN (no membership)", async () => { - // Membership check runs before the org lookup, so a missing org id - // surfaces as a FORBIDDEN rather than NOT_FOUND for callers who are - // not members. This matches the rest of the codebase's defense-in-depth. - const update = getUpdateHandler(); - await expect( - update( - { id: "019d694f-79f6-7595-8faf-b70b01c11f98", name: "Nope" }, - createContext(testIdentity), - ), - ).rejects.toThrow("Only owners and admins can update the organization"); - }); -}); diff --git a/packages/server/rpc/accounts/org.ts b/packages/server/rpc/accounts/org.ts deleted file mode 100644 index 2a93a78..0000000 --- a/packages/server/rpc/accounts/org.ts +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Accounts RPC org methods. - * - * Implements: - * - org.create: Create a new organization (caller becomes owner) - * - org.list: List organizations for the current identity - * - org.get: Get organization by ID - * - org.update: Update organization name - * - org.delete: Delete an organization - */ -import type { Org } from "@memory.build/accounts"; -import type { - OrgCreateParams, - OrgDeleteParams, - OrgGetParams, - OrgListParams, - OrgResponse, - OrgUpdateParams, -} from "@memory.build/protocol/accounts/org"; -import { - orgCreateParams, - orgDeleteParams, - orgGetParams, - orgListParams, - orgUpdateParams, -} from "@memory.build/protocol/accounts/org"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an Org to a serializable response. - */ -function toOrgResponse(org: Org): OrgResponse { - return { - id: org.id, - slug: org.slug, - name: org.name, - createdAt: org.createdAt.toISOString(), - updatedAt: org.updatedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * org.create - Create a new organization. - * The authenticated identity automatically becomes the owner. - */ -async function orgCreate( - params: OrgCreateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Create org and add creator as owner in a transaction - const org = await db.withTransaction(async (txDb) => { - const newOrg = await txDb.createOrg({ - name: params.name, - }); - - // Add creator as owner - await txDb.addMember(newOrg.id, identity.id, "owner"); - - return newOrg; - }); - - return toOrgResponse(org); -} - -/** - * org.list - List organizations for the current identity. - */ -async function orgList( - _params: OrgListParams, - context: HandlerContext, -): Promise<{ orgs: OrgResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - const orgs = await db.listOrgsByIdentity(identity.id); - return { orgs: orgs.map(toOrgResponse) }; -} - -/** - * org.get - Get organization by ID. - */ -async function orgGet( - params: OrgGetParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const member = await db.getMember(params.id, identity.id); - if (!member) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const org = await db.getOrg(params.id); - if (!org) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - return toOrgResponse(org); -} - -/** - * org.update - Update organization name. - * Requires owner or admin role. - */ -async function orgUpdate( - params: OrgUpdateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const member = await db.getMember(params.id, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can update the organization", - ); - } - - const updated = await db.updateOrg(params.id, { - name: params.name, - }); - - if (!updated) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - const org = await db.getOrg(params.id); - if (!org) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - return toOrgResponse(org); -} - -/** - * org.delete - Delete an organization. - * Requires owner role. Refuses if: - * - The org has any engines (delete engines first) - * - It's the caller's only owned org - */ -async function orgDelete( - params: OrgDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is an owner - const member = await db.getMember(params.id, identity.id); - if (!member || member.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can delete the organization"); - } - - // Refuse if the org still has engines - const engines = await db.listEnginesByOrg(params.id); - if (engines.length > 0) { - throw new AppError( - "CONFLICT", - "Cannot delete organization with engines. Delete all engines first.", - ); - } - - // Refuse if this is the caller's only owned org - const ownedCount = await db.countOwnedOrgs(identity.id); - if (ownedCount <= 1) { - throw new AppError("CONFLICT", "Cannot delete your only organization."); - } - - const deleted = await db.deleteOrg(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - return { deleted }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the org methods registry. - */ -export const orgMethods = buildRegistry() - .register("org.create", orgCreateParams, orgCreate) - .register("org.list", orgListParams, orgList) - .register("org.get", orgGetParams, orgGet) - .register("org.update", orgUpdateParams, orgUpdate) - .register("org.delete", orgDeleteParams, orgDelete) - .build(); diff --git a/packages/server/rpc/accounts/schemas.test.ts b/packages/server/rpc/accounts/schemas.test.ts deleted file mode 100644 index 04f8287..0000000 --- a/packages/server/rpc/accounts/schemas.test.ts +++ /dev/null @@ -1,513 +0,0 @@ -/** - * Tests for Accounts RPC schemas. - */ -import { describe, expect, test } from "bun:test"; -import { - emailSchema, - engineCreateSchema, - engineGetSchema, - engineListSchema, - engineSetupAccessSchema, - engineStatusSchema, - engineUpdateSchema, - identityGetByEmailSchema, - invitationAcceptSchema, - invitationCreateSchema, - invitationListSchema, - invitationRevokeSchema, - meGetSchema, - nameSchema, - orgCreateSchema, - orgDeleteSchema, - orgGetSchema, - orgListSchema, - orgMemberAddSchema, - orgMemberListSchema, - orgMemberRemoveSchema, - orgMemberUpdateRoleSchema, - orgRoleSchema, - orgUpdateSchema, - uuidv7Schema, -} from "./schemas"; - -// ============================================================================= -// Common Schema Tests -// ============================================================================= - -describe("uuidv7Schema", () => { - test("accepts valid UUIDv7", () => { - const validUuids = [ - "019d694f-79f6-7595-8faf-b70b01c11f98", - "019d694f-79f6-7595-9faf-b70b01c11f98", - "019d694f-79f6-7595-afaf-b70b01c11f98", - "019d694f-79f6-7595-bfaf-b70b01c11f98", - ]; - for (const uuid of validUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(true); - } - }); - - test("rejects invalid UUIDs", () => { - const invalidUuids = [ - "not-a-uuid", - "019d694f-79f6-4595-8faf-b70b01c11f98", // v4 not v7 - "019d694f-79f6-7595-0faf-b70b01c11f98", // invalid variant - "019d694f79f675958fafb70b01c11f98", // no dashes - "", - ]; - for (const uuid of invalidUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(false); - } - }); -}); - -describe("emailSchema", () => { - test("accepts valid emails", () => { - const validEmails = [ - "user@example.com", - "user.name@example.com", - "user+tag@example.com", - "user@subdomain.example.com", - ]; - for (const email of validEmails) { - expect(emailSchema.safeParse(email).success).toBe(true); - } - }); - - test("rejects invalid emails", () => { - const invalidEmails = ["not-an-email", "user@", "@example.com", ""]; - for (const email of invalidEmails) { - expect(emailSchema.safeParse(email).success).toBe(false); - } - }); -}); - -describe("nameSchema", () => { - test("accepts valid names", () => { - const validNames = ["a", "John Doe", "My Organization", "a".repeat(100)]; - for (const name of validNames) { - expect(nameSchema.safeParse(name).success).toBe(true); - } - }); - - test("rejects invalid names", () => { - const invalidNames = ["", "a".repeat(101)]; - for (const name of invalidNames) { - expect(nameSchema.safeParse(name).success).toBe(false); - } - }); -}); - -describe("orgRoleSchema", () => { - test("accepts valid roles", () => { - const validRoles = ["owner", "admin", "member"]; - for (const role of validRoles) { - expect(orgRoleSchema.safeParse(role).success).toBe(true); - } - }); - - test("rejects invalid roles", () => { - const invalidRoles = ["superuser", "guest", "", "OWNER"]; - for (const role of invalidRoles) { - expect(orgRoleSchema.safeParse(role).success).toBe(false); - } - }); -}); - -describe("engineStatusSchema", () => { - test("accepts valid statuses", () => { - const validStatuses = ["active", "suspended", "deleted"]; - for (const status of validStatuses) { - expect(engineStatusSchema.safeParse(status).success).toBe(true); - } - }); - - test("rejects invalid statuses", () => { - const invalidStatuses = ["pending", "inactive", "", "ACTIVE"]; - for (const status of invalidStatuses) { - expect(engineStatusSchema.safeParse(status).success).toBe(false); - } - }); -}); - -// ============================================================================= -// Me Schema Tests -// ============================================================================= - -describe("identityGetByEmailSchema", () => { - test("accepts valid email", () => { - const result = identityGetByEmailSchema.safeParse({ - email: "user@example.com", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid email", () => { - const result = identityGetByEmailSchema.safeParse({ - email: "not-an-email", - }); - expect(result.success).toBe(false); - }); - - test("rejects missing email", () => { - const result = identityGetByEmailSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -describe("meGetSchema", () => { - test("accepts empty params", () => { - const result = meGetSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("ignores extra params", () => { - const result = meGetSchema.safeParse({ extra: "ignored" }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Org Schema Tests -// ============================================================================= - -describe("orgCreateSchema", () => { - test("accepts valid params", () => { - const result = orgCreateSchema.safeParse({ - name: "My Organization", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = orgCreateSchema.safeParse({ - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("orgListSchema", () => { - test("accepts empty params", () => { - const result = orgListSchema.safeParse({}); - expect(result.success).toBe(true); - }); -}); - -describe("orgGetSchema", () => { - test("accepts valid UUID", () => { - const result = orgGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid UUID", () => { - const result = orgGetSchema.safeParse({ - id: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); -}); - -describe("orgUpdateSchema", () => { - test("accepts id with name update", () => { - const result = orgUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "New Name", - }); - expect(result.success).toBe(true); - }); - - test("accepts id only (no-op)", () => { - const result = orgUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("orgDeleteSchema", () => { - test("accepts valid UUID", () => { - const result = orgDeleteSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Org Member Schema Tests -// ============================================================================= - -describe("orgMemberListSchema", () => { - test("accepts valid orgId", () => { - const result = orgMemberListSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("orgMemberAddSchema", () => { - test("accepts valid params", () => { - const result = orgMemberAddSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "member", - }); - expect(result.success).toBe(true); - }); - - test("accepts all roles", () => { - for (const role of ["owner", "admin", "member"]) { - const result = orgMemberAddSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role, - }); - expect(result.success).toBe(true); - } - }); - - test("rejects invalid role", () => { - const result = orgMemberAddSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "superuser", - }); - expect(result.success).toBe(false); - }); -}); - -describe("orgMemberRemoveSchema", () => { - test("accepts valid params", () => { - const result = orgMemberRemoveSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - }); - expect(result.success).toBe(true); - }); -}); - -describe("orgMemberUpdateRoleSchema", () => { - test("accepts valid params", () => { - const result = orgMemberUpdateRoleSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "admin", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Engine Schema Tests -// ============================================================================= - -describe("engineCreateSchema", () => { - test("accepts valid params", () => { - const result = engineCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "My Engine", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = engineCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("engineListSchema", () => { - test("accepts valid orgId", () => { - const result = engineListSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("engineGetSchema", () => { - test("accepts valid UUID", () => { - const result = engineGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("engineUpdateSchema", () => { - test("accepts name update", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "New Engine Name", - }); - expect(result.success).toBe(true); - }); - - test("accepts status update", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - status: "suspended", - }); - expect(result.success).toBe(true); - }); - - test("accepts id only (no-op)", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid status", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - status: "invalid", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Invitation Schema Tests -// ============================================================================= - -describe("invitationCreateSchema", () => { - test("accepts minimal params", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "member", - }); - expect(result.success).toBe(true); - }); - - test("accepts with expiresInDays", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "admin", - expiresInDays: 14, - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid email", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "not-an-email", - role: "member", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid role", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "superuser", - }); - expect(result.success).toBe(false); - }); - - test("rejects expiresInDays < 1", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "member", - expiresInDays: 0, - }); - expect(result.success).toBe(false); - }); - - test("rejects expiresInDays > 30", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "member", - expiresInDays: 31, - }); - expect(result.success).toBe(false); - }); -}); - -describe("invitationListSchema", () => { - test("accepts valid orgId", () => { - const result = invitationListSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("invitationRevokeSchema", () => { - test("accepts valid UUID", () => { - const result = invitationRevokeSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("invitationAcceptSchema", () => { - test("accepts valid token", () => { - const result = invitationAcceptSchema.safeParse({ - token: "some-invitation-token-here", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty token", () => { - const result = invitationAcceptSchema.safeParse({ - token: "", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Engine SetupAccess Schema Tests -// ============================================================================= - -describe("engineSetupAccessSchema", () => { - test("accepts valid engineId only", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("accepts engineId with apiKeyName", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "019d694f-79f6-7595-8faf-b70b01c11f98", - apiKeyName: "my-cli-key", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing engineId", () => { - const result = engineSetupAccessSchema.safeParse({}); - expect(result.success).toBe(false); - }); - - test("rejects invalid UUID", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); - - test("rejects empty apiKeyName", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "019d694f-79f6-7595-8faf-b70b01c11f98", - apiKeyName: "", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/server/rpc/accounts/schemas.ts b/packages/server/rpc/accounts/schemas.ts deleted file mode 100644 index 5c3b7b1..0000000 --- a/packages/server/rpc/accounts/schemas.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Re-export accounts schemas from @memory.build/protocol. - * - * @deprecated Import directly from @memory.build/protocol/accounts instead. - */ - -export { - type EngineCreateParams, - type EngineGetParams, - type EngineListParams, - type EngineSetupAccessParams, - type EngineUpdateParams, - // Engine params - engineCreateParams as engineCreateSchema, - engineGetParams as engineGetSchema, - engineListParams as engineListSchema, - engineSetupAccessParams as engineSetupAccessSchema, - engineUpdateParams as engineUpdateSchema, -} from "@memory.build/protocol/accounts/engine"; - -export { - type IdentityGetByEmailParams, - identityGetByEmailParams as identityGetByEmailSchema, - type MeGetParams, - // Identity params - meGetParams as meGetSchema, -} from "@memory.build/protocol/accounts/identity"; -export { - type InvitationAcceptParams, - type InvitationCreateParams, - type InvitationListParams, - type InvitationRevokeParams, - invitationAcceptParams as invitationAcceptSchema, - // Invitation params - invitationCreateParams as invitationCreateSchema, - invitationListParams as invitationListSchema, - invitationRevokeParams as invitationRevokeSchema, -} from "@memory.build/protocol/accounts/invitation"; - -export { - type OrgCreateParams, - type OrgDeleteParams, - type OrgGetParams, - type OrgListParams, - type OrgUpdateParams, - // Org params - orgCreateParams as orgCreateSchema, - orgDeleteParams as orgDeleteSchema, - orgGetParams as orgGetSchema, - orgListParams as orgListSchema, - orgUpdateParams as orgUpdateSchema, -} from "@memory.build/protocol/accounts/org"; - -export { - type OrgMemberAddParams, - type OrgMemberListParams, - type OrgMemberRemoveParams, - type OrgMemberUpdateRoleParams, - orgMemberAddParams as orgMemberAddSchema, - // Org member params - orgMemberListParams as orgMemberListSchema, - orgMemberRemoveParams as orgMemberRemoveSchema, - orgMemberUpdateRoleParams as orgMemberUpdateRoleSchema, -} from "@memory.build/protocol/accounts/org-member"; -export { - type SessionRevokeParams, - // Session params - sessionRevokeParams as sessionRevokeSchema, -} from "@memory.build/protocol/accounts/session"; -export { - // Fields - emailSchema, - engineStatusSchema, - nameSchema, - orgRoleSchema, - uuidv7Schema, -} from "@memory.build/protocol/fields"; diff --git a/packages/server/rpc/accounts/session.ts b/packages/server/rpc/accounts/session.ts deleted file mode 100644 index 64ee4da..0000000 --- a/packages/server/rpc/accounts/session.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Accounts RPC session methods. - * - * Implements: - * - session.revoke: Revoke the current session (logout) - */ -import type { SessionRevokeParams } from "@memory.build/protocol/accounts/session"; -import { sessionRevokeParams } from "@memory.build/protocol/accounts/session"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * session.revoke - Revoke the current session (logout). - * - * This deletes all sessions for the authenticated identity, - * effectively logging the user out of all devices. - * - * TODO: If we need per-session revocation, pass sessionId through context. - */ -async function sessionRevoke( - _params: SessionRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertAccountsRpcContext(context); - const { auth, identity } = context as AccountsRpcContext; - - const count = await auth.deleteSessionsByUser(identity.id); - return { revoked: count > 0 }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the session methods registry. - */ -export const sessionMethods = buildRegistry() - .register("session.revoke", sessionRevokeParams, sessionRevoke) - .build(); diff --git a/packages/server/rpc/accounts/types.ts b/packages/server/rpc/accounts/types.ts deleted file mode 100644 index 4081b19..0000000 --- a/packages/server/rpc/accounts/types.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Accounts RPC context types. - * - * Extends the base HandlerContext with accounts-specific fields. - */ -import type { AccountsDB } from "@memory.build/accounts"; -import type { AuthStore } from "@memory.build/auth"; -import type { SQL } from "bun"; -import type { Identity } from "../../middleware/authenticate"; -import type { HandlerContext } from "../types"; - -/** - * Accounts RPC handler context. - * - * Provides access to: - * - `db`: AccountsDB instance for accounts operations - * - `identity`: The authenticated identity (from OAuth session) - * - `engineSql`: SQL connection to the engine database (for schema provisioning) - * - `serverVersion`: Application version string (for migration tracking) - * - * Authentication middleware populates these fields via OAuth session validation. - */ -export interface AccountsRpcContext extends HandlerContext { - /** Legacy AccountsDB instance (org/engine/member/invitation, until Phase 5) */ - db: AccountsDB; - /** Auth store (auth schema): me/session/identity */ - auth: AuthStore; - /** Authenticated user (session-validated) */ - identity: Identity; - /** SQL connection to the engine database */ - engineSql: SQL; - /** Application version string */ - serverVersion: string; -} - -/** - * Type guard to check if context has accounts fields. - */ -export function isAccountsRpcContext( - ctx: HandlerContext, -): ctx is AccountsRpcContext { - return ( - "db" in ctx && - typeof ctx.db === "object" && - ctx.db !== null && - "auth" in ctx && - typeof ctx.auth === "object" && - ctx.auth !== null && - "identity" in ctx && - typeof ctx.identity === "object" && - ctx.identity !== null && - "id" in ctx.identity && - "email" in ctx.identity && - "engineSql" in ctx && - typeof ctx.engineSql === "function" && - "serverVersion" in ctx && - typeof ctx.serverVersion === "string" - ); -} - -/** - * Assert that context is an AccountsRpcContext, throwing if not. - */ -export function assertAccountsRpcContext( - ctx: HandlerContext, -): asserts ctx is AccountsRpcContext { - if (!isAccountsRpcContext(ctx)) { - throw new Error( - "Accounts context not initialized (authentication required)", - ); - } -} diff --git a/packages/server/rpc/engine/api-key.ts b/packages/server/rpc/engine/api-key.ts deleted file mode 100644 index 7bc946e..0000000 --- a/packages/server/rpc/engine/api-key.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Engine RPC API key methods. - * - * Implements: - * - apiKey.create: Create a new API key (returns raw key once) - * - apiKey.get: Get API key metadata by ID - * - apiKey.list: List API keys for a user - * - apiKey.revoke: Revoke an API key - * - apiKey.delete: Permanently delete an API key - */ -import type { ApiKey } from "@memory.build/engine"; -import type { - ApiKeyCreateParams, - ApiKeyCreateResult, - ApiKeyDeleteParams, - ApiKeyGetParams, - ApiKeyListParams, - ApiKeyResponse, - ApiKeyRevokeParams, -} from "@memory.build/protocol/engine/api-key"; -import { - apiKeyCreateParams, - apiKeyDeleteParams, - apiKeyGetParams, - apiKeyListParams, - apiKeyRevokeParams, -} from "@memory.build/protocol/engine/api-key"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert an ApiKey to a serializable response. - */ -function toApiKeyResponse(apiKey: ApiKey): ApiKeyResponse { - return { - id: apiKey.id, - userId: apiKey.userId, - lookupId: apiKey.lookupId, - name: apiKey.name, - expiresAt: apiKey.expiresAt?.toISOString() ?? null, - createdAt: apiKey.createdAt.toISOString(), - revokedAt: apiKey.revokedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * apiKey.create - Create a new API key. - * Returns the raw key once - it cannot be retrieved again. - */ -async function apiKeyCreate( - params: ApiKeyCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const result = await db.createApiKey({ - userId: params.userId, - name: params.name, - expiresAt: params.expiresAt ? new Date(params.expiresAt) : undefined, - }); - - return { - apiKey: toApiKeyResponse(result.apiKey), - rawKey: result.rawKey, - }; -} - -/** - * apiKey.get - Get API key metadata by ID. - */ -async function apiKeyGet( - params: ApiKeyGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const apiKey = await db.getApiKey(params.id); - if (!apiKey) { - throw new AppError("NOT_FOUND", `API key not found: ${params.id}`); - } - - return toApiKeyResponse(apiKey); -} - -/** - * apiKey.list - List API keys for a user. - */ -async function apiKeyList( - params: ApiKeyListParams, - context: HandlerContext, -): Promise<{ apiKeys: ApiKeyResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const apiKeys = await db.listApiKeys(params.userId); - return { apiKeys: apiKeys.map(toApiKeyResponse) }; -} - -/** - * apiKey.revoke - Revoke an API key (soft delete). - */ -async function apiKeyRevoke( - params: ApiKeyRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const revoked = await db.revokeApiKey(params.id); - if (!revoked) { - throw new AppError( - "NOT_FOUND", - `API key not found or already revoked: ${params.id}`, - ); - } - - return { revoked }; -} - -/** - * apiKey.delete - Permanently delete an API key. - */ -async function apiKeyDelete( - params: ApiKeyDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const deleted = await db.deleteApiKey(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `API key not found: ${params.id}`); - } - - return { deleted }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the API key methods registry. - */ -export const apiKeyMethods = buildRegistry() - .register("apiKey.create", apiKeyCreateParams, apiKeyCreate) - .register("apiKey.get", apiKeyGetParams, apiKeyGet) - .register("apiKey.list", apiKeyListParams, apiKeyList) - .register("apiKey.revoke", apiKeyRevokeParams, apiKeyRevoke) - .register("apiKey.delete", apiKeyDeleteParams, apiKeyDelete) - .build(); diff --git a/packages/server/rpc/engine/grant.test.ts b/packages/server/rpc/engine/grant.test.ts deleted file mode 100644 index c2335f8..0000000 --- a/packages/server/rpc/engine/grant.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -/** - * Unit tests for grant RPC handlers. - * - * Uses mocked EngineDB to test handler logic in isolation. - * Verifies that responses include userName from JOINed user data. - */ -import { describe, expect, mock, test } from "bun:test"; -import type { HandlerContext } from "../types"; -import { grantMethods } from "./grant"; - -const TEST_UUID = "019d694f-79f6-7595-8faf-b70b01c11f98"; -const TEST_UUID_2 = "019d694f-79f6-7595-8faf-b70b01c11f99"; - -function createMockContext( - dbOverrides: Record = {}, -): HandlerContext { - return { - request: new Request("http://localhost"), - db: { - grantTreeAccess: mock(() => Promise.resolve()), - revokeTreeAccess: mock(() => Promise.resolve(false)), - listTreeGrants: mock(() => Promise.resolve([])), - getTreeGrant: mock(() => Promise.resolve(null)), - checkTreeAccess: mock(() => Promise.resolve(false)), - ...dbOverrides, - }, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; -} - -// ============================================================================= -// grant.create -// ============================================================================= - -describe("grant.create", () => { - test("calls grantTreeAccess and returns { created: true }", async () => { - const handler = grantMethods.get("grant.create")?.handler; - if (!handler) throw new Error("grant.create handler not found"); - - const grantTreeAccess = mock(() => Promise.resolve()); - const context = createMockContext({ grantTreeAccess }); - - const result = await handler( - { - userId: TEST_UUID, - treePath: "work.projects", - actions: ["read", "create"], - withGrantOption: false, - }, - context, - ); - - expect(result).toEqual({ created: true }); - expect(grantTreeAccess).toHaveBeenCalledTimes(1); - }); -}); - -// ============================================================================= -// grant.list -// ============================================================================= - -describe("grant.list", () => { - test("returns grants with userName", async () => { - const handler = grantMethods.get("grant.list")?.handler; - if (!handler) throw new Error("grant.list handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const listTreeGrants = mock(() => - Promise.resolve([ - { - id: TEST_UUID_2, - userId: TEST_UUID, - userName: "alice", - treePath: "work.projects", - actions: ["read"], - grantedBy: null, - withGrantOption: false, - createdAt: now, - }, - ]), - ); - const context = createMockContext({ listTreeGrants }); - - const result = (await handler({}, context)) as { - grants: Array<{ - userId: string; - userName: string; - treePath: string; - }>; - }; - - expect(result.grants).toHaveLength(1); - expect(result.grants[0]?.userName).toBe("alice"); - expect(result.grants[0]?.userId).toBe(TEST_UUID); - expect(result.grants[0]?.treePath).toBe("work.projects"); - }); - - test("returns empty list when no grants", async () => { - const handler = grantMethods.get("grant.list")?.handler; - if (!handler) throw new Error("grant.list handler not found"); - - const context = createMockContext(); - - const result = (await handler({}, context)) as { - grants: unknown[]; - }; - - expect(result.grants).toHaveLength(0); - }); - - test("passes userId filter when provided", async () => { - const handler = grantMethods.get("grant.list")?.handler; - if (!handler) throw new Error("grant.list handler not found"); - - const listTreeGrants = mock(() => Promise.resolve([])); - const context = createMockContext({ listTreeGrants }); - - await handler({ userId: TEST_UUID }, context); - - expect(listTreeGrants).toHaveBeenCalledWith(TEST_UUID); - }); -}); - -// ============================================================================= -// grant.get -// ============================================================================= - -describe("grant.get", () => { - test("returns grant with userName when found", async () => { - const handler = grantMethods.get("grant.get")?.handler; - if (!handler) throw new Error("grant.get handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const getTreeGrant = mock(() => - Promise.resolve({ - id: TEST_UUID_2, - userId: TEST_UUID, - userName: "alice", - treePath: "work", - actions: ["read", "create"], - grantedBy: null, - withGrantOption: true, - createdAt: now, - }), - ); - const context = createMockContext({ getTreeGrant }); - - const result = (await handler( - { userId: TEST_UUID, treePath: "work" }, - context, - )) as { userName: string; withGrantOption: boolean }; - - expect(result.userName).toBe("alice"); - expect(result.withGrantOption).toBe(true); - }); - - test("throws NOT_FOUND when grant does not exist", async () => { - const handler = grantMethods.get("grant.get")?.handler; - if (!handler) throw new Error("grant.get handler not found"); - - const context = createMockContext({ - getTreeGrant: mock(() => Promise.resolve(null)), - }); - - try { - await handler({ userId: TEST_UUID, treePath: "work" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -// ============================================================================= -// grant.revoke -// ============================================================================= - -describe("grant.revoke", () => { - test("returns { revoked: true } when found", async () => { - const handler = grantMethods.get("grant.revoke")?.handler; - if (!handler) throw new Error("grant.revoke handler not found"); - - const context = createMockContext({ - revokeTreeAccess: mock(() => Promise.resolve(true)), - }); - - const result = await handler( - { userId: TEST_UUID, treePath: "work" }, - context, - ); - expect(result).toEqual({ revoked: true }); - }); - - test("throws NOT_FOUND when grant does not exist", async () => { - const handler = grantMethods.get("grant.revoke")?.handler; - if (!handler) throw new Error("grant.revoke handler not found"); - - const context = createMockContext({ - revokeTreeAccess: mock(() => Promise.resolve(false)), - }); - - try { - await handler({ userId: TEST_UUID, treePath: "work" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -// ============================================================================= -// grant.check -// ============================================================================= - -describe("grant.check", () => { - test("returns { allowed: true } when access granted", async () => { - const handler = grantMethods.get("grant.check")?.handler; - if (!handler) throw new Error("grant.check handler not found"); - - const context = createMockContext({ - checkTreeAccess: mock(() => Promise.resolve(true)), - }); - - const result = await handler( - { userId: TEST_UUID, treePath: "work", action: "read" }, - context, - ); - expect(result).toEqual({ allowed: true }); - }); - - test("returns { allowed: false } when access denied", async () => { - const handler = grantMethods.get("grant.check")?.handler; - if (!handler) throw new Error("grant.check handler not found"); - - const context = createMockContext({ - checkTreeAccess: mock(() => Promise.resolve(false)), - }); - - const result = await handler( - { userId: TEST_UUID, treePath: "work", action: "update" }, - context, - ); - expect(result).toEqual({ allowed: false }); - }); -}); diff --git a/packages/server/rpc/engine/grant.ts b/packages/server/rpc/engine/grant.ts deleted file mode 100644 index ba44036..0000000 --- a/packages/server/rpc/engine/grant.ts +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Engine RPC grant methods. - * - * Implements: - * - grant.create: Grant tree access to a user - * - grant.list: List grants (optionally filter by user) - * - grant.get: Get a specific grant - * - grant.revoke: Revoke tree access - * - grant.check: Check if user has access to a tree path for an action - */ -import type { TreeGrant } from "@memory.build/engine"; -import type { - GrantCheckParams, - GrantCreateParams, - GrantGetParams, - GrantListParams, - GrantResponse, - GrantRevokeParams, -} from "@memory.build/protocol/engine/grant"; -import { - grantCheckParams, - grantCreateParams, - grantGetParams, - grantListParams, - grantRevokeParams, -} from "@memory.build/protocol/engine/grant"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a TreeGrant to a serializable response. - */ -function toGrantResponse(grant: TreeGrant): GrantResponse { - return { - id: grant.id, - userId: grant.userId, - userName: grant.userName, - treePath: grant.treePath, - actions: grant.actions, - grantedBy: grant.grantedBy, - withGrantOption: grant.withGrantOption, - createdAt: grant.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * grant.create - Grant tree access to a user. - */ -async function grantCreate( - params: GrantCreateParams, - context: HandlerContext, -): Promise<{ created: boolean }> { - assertEngineContext(context); - const { db, userId } = context as EngineContext; - - await db.grantTreeAccess({ - userId: params.userId, - treePath: params.treePath, - actions: params.actions, - grantedBy: userId, - withGrantOption: params.withGrantOption, - }); - - return { created: true }; -} - -/** - * grant.list - List grants. - */ -async function grantList( - params: GrantListParams, - context: HandlerContext, -): Promise<{ grants: GrantResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const grants = await db.listTreeGrants(params.userId); - return { grants: grants.map(toGrantResponse) }; -} - -/** - * grant.get - Get a specific grant. - */ -async function grantGet( - params: GrantGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const grant = await db.getTreeGrant(params.userId, params.treePath); - if (!grant) { - throw new AppError( - "NOT_FOUND", - `Grant not found for user ${params.userId} at path ${params.treePath}`, - ); - } - - return toGrantResponse(grant); -} - -/** - * grant.revoke - Revoke tree access. - */ -async function grantRevoke( - params: GrantRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const revoked = await db.revokeTreeAccess(params.userId, params.treePath); - if (!revoked) { - throw new AppError( - "NOT_FOUND", - `Grant not found for user ${params.userId} at path ${params.treePath}`, - ); - } - - return { revoked }; -} - -/** - * grant.check - Check if user has access to a tree path for an action. - */ -async function grantCheck( - params: GrantCheckParams, - context: HandlerContext, -): Promise<{ allowed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const allowed = await db.checkTreeAccess( - params.userId, - params.treePath, - params.action, - ); - - return { allowed }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the grant methods registry. - */ -export const grantMethods = buildRegistry() - .register("grant.create", grantCreateParams, grantCreate) - .register("grant.list", grantListParams, grantList) - .register("grant.get", grantGetParams, grantGet) - .register("grant.revoke", grantRevokeParams, grantRevoke) - .register("grant.check", grantCheckParams, grantCheck) - .build(); diff --git a/packages/server/rpc/engine/index.ts b/packages/server/rpc/engine/index.ts deleted file mode 100644 index f4b2b28..0000000 --- a/packages/server/rpc/engine/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { buildRegistry } from "../registry"; -import { apiKeyMethods } from "./api-key"; -import { grantMethods } from "./grant"; -import { memoryMethods } from "./memory"; -import { ownerMethods } from "./owner"; -import { roleMethods } from "./role"; -import { userMethods } from "./user"; - -/** - * Engine RPC method registry. - * - * Memory methods (chunk 3): - * - memory.create, memory.batchCreate, memory.get, memory.update, memory.delete - * - memory.search, memory.tree, memory.move, memory.deleteTree, memory.countTree - * - * User, grant, role methods (chunk 4): - * - user.create, user.get, user.getByName, user.list, user.rename, user.delete - * - grant.create, grant.list, grant.get, grant.revoke, grant.check - * - role.create, role.addMember, role.removeMember, role.listMembers, role.listForUser - * - * API key methods (chunk 5): - * - apiKey.create, apiKey.get, apiKey.list, apiKey.revoke, apiKey.delete - */ -export const engineMethods = buildRegistry() - .merge(memoryMethods) - .merge(userMethods) - .merge(grantMethods) - .merge(ownerMethods) - .merge(roleMethods) - .merge(apiKeyMethods) - .build(); - -// Re-export types for consumers -export type { EngineContext } from "./types"; -export { assertEngineContext, isEngineContext } from "./types"; diff --git a/packages/server/rpc/engine/memory.test.ts b/packages/server/rpc/engine/memory.test.ts deleted file mode 100644 index c4887b3..0000000 --- a/packages/server/rpc/engine/memory.test.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { describe, expect, mock, test } from "bun:test"; -import type { HandlerContext } from "../types"; - -describe("memory.search embedding", () => { - test("throws EMBEDDING_NOT_CONFIGURED when semantic provided without config", async () => { - // Import the handler module to test - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockDb = { - searchMemories: mock(() => - Promise.resolve({ results: [], total: 0, limit: 10 }), - ), - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - // embeddingConfig intentionally omitted - } as unknown as HandlerContext; - - const params = { - semantic: "test query", - }; - - try { - await handler(params, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("EMBEDDING_NOT_CONFIGURED"); - } - }); - - test("throws EMBEDDING_FAILED when embedding generation fails", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockDb = { - searchMemories: mock(() => - Promise.resolve({ results: [], total: 0, limit: 10 }), - ), - }; - - const embeddingConfig = { - provider: "openai" as const, - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - embeddingConfig, - } as unknown as HandlerContext; - - const params = { - semantic: "test query", - }; - - // The actual embedding call will fail because we're using a fake API key - // This tests that errors are properly caught and wrapped - try { - await handler(params, context); - // If embedding somehow succeeds (unlikely with fake key), that's fine too - } catch (error) { - // Should be wrapped in AppError with EMBEDDING_FAILED code - expect((error as { code: string }).code).toBe("EMBEDDING_FAILED"); - } - }); - - test("calls searchMemories without embedding when semantic not provided", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockSearchMemories = mock(() => - Promise.resolve({ - results: [ - { - id: "mem-1", - content: "test", - score: 1.0, - meta: {}, - tree: "", - temporal: null, - hasEmbedding: false, - createdAt: new Date(), - createdBy: null, - updatedAt: null, - }, - ], - total: 1, - limit: 10, - }), - ); - - const mockDb = { - searchMemories: mockSearchMemories, - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - // No embeddingConfig needed when not using semantic - } as unknown as HandlerContext; - - const params = { - fulltext: "test query", - }; - - await handler(params, context); - - // Verify searchMemories was called without embedding - expect(mockSearchMemories).toHaveBeenCalled(); - const calls = mockSearchMemories.mock.calls as unknown as Array< - [{ fulltext?: string; embedding?: number[] }] - >; - expect(calls.length).toBeGreaterThan(0); - const callArgs = calls[0]![0]!; - expect(callArgs.fulltext).toBe("test query"); - expect(callArgs.embedding).toBeUndefined(); - }); - - test("passes grep parameter through to searchMemories", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockSearchMemories = mock(() => - Promise.resolve({ - results: [], - total: 0, - limit: 10, - }), - ); - - const mockDb = { - searchMemories: mockSearchMemories, - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; - - const params = { - fulltext: "database", - grep: "version \\d+", - }; - - await handler(params, context); - - expect(mockSearchMemories).toHaveBeenCalled(); - const calls = mockSearchMemories.mock.calls as unknown as Array< - [{ fulltext?: string; grep?: string }] - >; - expect(calls.length).toBeGreaterThan(0); - const callArgs = calls[0]![0]!; - expect(callArgs.fulltext).toBe("database"); - expect(callArgs.grep).toBe("version \\d+"); - }); - - test("throws VALIDATION_ERROR when grep is used alone", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockDb = { - searchMemories: mock(() => - Promise.resolve({ results: [], total: 0, limit: 10 }), - ), - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; - - try { - await handler({ grep: "ERR-\\d+" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("VALIDATION_ERROR"); - } - }); -}); diff --git a/packages/server/rpc/engine/memory.ts b/packages/server/rpc/engine/memory.ts deleted file mode 100644 index c877b34..0000000 --- a/packages/server/rpc/engine/memory.ts +++ /dev/null @@ -1,414 +0,0 @@ -/** - * Engine RPC memory methods. - * - * Implements: - * - memory.create: Create a single memory - * - memory.batchCreate: Create multiple memories - * - memory.get: Get memory by ID - * - memory.update: Update memory content/meta/tree/temporal - * - memory.delete: Delete memory by ID - * - memory.search: Hybrid semantic + fulltext search - * - memory.tree: Get tree structure with counts - * - memory.move: Move memories from one tree path to another - * - memory.deleteTree: Delete all memories under a tree path - */ -import { generateEmbedding } from "@memory.build/embedding"; -import type { Memory, SearchResult, TreeNode } from "@memory.build/engine"; -import type { - MemoryBatchCreateParams, - MemoryCountTreeParams, - MemoryCreateParams, - MemoryDeleteParams, - MemoryDeleteTreeParams, - MemoryGetParams, - MemoryMoveParams, - MemoryResponse, - MemorySearchParams, - MemorySearchResult, - MemoryTreeParams, - MemoryUpdateParams, -} from "@memory.build/protocol/engine/memory"; -import { - memoryBatchCreateParams, - memoryCountTreeParams, - memoryCreateParams, - memoryDeleteParams, - memoryDeleteTreeParams, - memoryGetParams, - memoryMoveParams, - memorySearchParams, - memoryTreeParams, - memoryUpdateParams, -} from "@memory.build/protocol/engine/memory"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a Memory to a serializable response. - */ -function toMemoryResponse(memory: Memory): MemoryResponse { - return { - id: memory.id, - content: memory.content, - meta: memory.meta, - tree: memory.tree, - temporal: memory.temporal - ? { - start: memory.temporal.start.toISOString(), - end: memory.temporal.end.toISOString(), - } - : null, - hasEmbedding: memory.hasEmbedding, - createdAt: memory.createdAt.toISOString(), - createdBy: memory.createdBy, - updatedAt: memory.updatedAt?.toISOString() ?? null, - }; -} - -/** - * Convert SearchResult to serializable response. - */ -function toSearchResultResponse(result: SearchResult): MemorySearchResult { - return { - results: result.results.map((item) => ({ - ...toMemoryResponse(item), - score: item.score, - })), - total: result.total, - limit: result.limit, - }; -} - -// ============================================================================= -// Helper Functions -// ============================================================================= - -/** - * Parse temporal params into Date objects. - */ -function parseTemporal( - temporal: { start: string; end?: string | null } | null | undefined, -): { start: Date; end?: Date } | undefined { - if (!temporal) return undefined; - return { - start: new Date(temporal.start), - end: temporal.end ? new Date(temporal.end) : undefined, - }; -} - -/** - * Parse temporal filter params into the format expected by engine ops. - */ -function parseTemporalFilter( - temporal: - | { - contains?: string; - overlaps?: { start: string; end: string }; - within?: { start: string; end: string }; - } - | null - | undefined, -): - | { - contains?: Date; - overlaps?: [Date, Date]; - within?: [Date, Date]; - } - | undefined { - if (!temporal) return undefined; - - const result: { - contains?: Date; - overlaps?: [Date, Date]; - within?: [Date, Date]; - } = {}; - - if (temporal.contains) { - result.contains = new Date(temporal.contains); - } - if (temporal.overlaps) { - result.overlaps = [ - new Date(temporal.overlaps.start), - new Date(temporal.overlaps.end), - ]; - } - if (temporal.within) { - result.within = [ - new Date(temporal.within.start), - new Date(temporal.within.end), - ]; - } - - return result; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * memory.create - Create a single memory. - */ -async function memoryCreate( - params: MemoryCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db, userId } = context as EngineContext; - - const memory = await db.createMemory({ - id: params.id ?? undefined, - content: params.content, - meta: params.meta ?? undefined, - tree: params.tree ?? undefined, - temporal: parseTemporal(params.temporal), - createdBy: userId, - }); - - return toMemoryResponse(memory); -} - -/** - * memory.batchCreate - Create multiple memories. - */ -async function memoryBatchCreate( - params: MemoryBatchCreateParams, - context: HandlerContext, -): Promise<{ ids: string[] }> { - assertEngineContext(context); - const { db, userId } = context as EngineContext; - - const ids = await db.batchCreateMemories( - params.memories.map( - (m: { - id?: string | null; - content: string; - meta?: Record | null; - tree?: string | null; - temporal?: { start: string; end?: string | null } | null; - }) => ({ - id: m.id ?? undefined, - content: m.content, - meta: m.meta ?? undefined, - tree: m.tree ?? undefined, - temporal: parseTemporal(m.temporal), - createdBy: userId, - }), - ), - ); - - return { ids }; -} - -/** - * memory.get - Get memory by ID. - */ -async function memoryGet( - params: MemoryGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const memory = await db.getMemory(params.id); - if (!memory) { - throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); - } - - return toMemoryResponse(memory); -} - -/** - * memory.update - Update memory content/meta/tree/temporal. - */ -async function memoryUpdate( - params: MemoryUpdateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const memory = await db.updateMemory(params.id, { - content: params.content ?? undefined, - meta: params.meta ?? undefined, - tree: params.tree ?? undefined, - temporal: parseTemporal(params.temporal), - }); - - if (!memory) { - throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); - } - - return toMemoryResponse(memory); -} - -/** - * memory.delete - Delete memory by ID. - */ -async function memoryDelete( - params: MemoryDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const deleted = await db.deleteMemory(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); - } - - return { deleted }; -} - -/** - * memory.search - Hybrid semantic + fulltext search. - */ -async function memorySearch( - params: MemorySearchParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db, embeddingConfig } = context as EngineContext; - - let embedding: number[] | undefined; - - // Generate embedding for semantic search - if (params.semantic) { - if (!embeddingConfig) { - throw new AppError( - "EMBEDDING_NOT_CONFIGURED", - "Semantic search requires embedding configuration. Set EMBEDDING_API_KEY environment variable.", - ); - } - - try { - const result = await generateEmbedding(params.semantic, embeddingConfig); - embedding = result.embedding; - } catch (error) { - throw new AppError( - "EMBEDDING_FAILED", - `Failed to generate embedding: ${error instanceof Error ? error.message : "Unknown error"}`, - ); - } - } - - // grep alone would cause a full table scan — require at least one indexed criterion - if ( - params.grep && - !params.fulltext && - !params.semantic && - !params.tree && - !params.meta && - !params.temporal - ) { - throw new AppError( - "VALIDATION_ERROR", - "grep cannot be used alone (full table scan). Combine with semantic, fulltext, tree, meta, or temporal.", - ); - } - - const result = await db.searchMemories({ - fulltext: params.fulltext ?? undefined, - embedding, - grep: params.grep ?? undefined, - tree: params.tree ?? undefined, - meta: params.meta ?? undefined, - temporal: parseTemporalFilter(params.temporal), - limit: params.limit, - candidateLimit: params.candidateLimit, - semanticThreshold: params.semanticThreshold ?? undefined, - weights: params.weights ?? undefined, - orderBy: params.orderBy, - }); - - return toSearchResultResponse(result); -} - -/** - * memory.tree - Get tree structure with counts. - */ -async function memoryTree( - params: MemoryTreeParams, - context: HandlerContext, -): Promise<{ nodes: TreeNode[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const nodes = await db.getTree({ - tree: params.tree ?? undefined, - levels: params.levels, - }); - - return { nodes }; -} - -/** - * memory.move - Move memories from one tree path to another. - */ -async function memoryMove( - params: MemoryMoveParams, - context: HandlerContext, -): Promise<{ count: number }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - if (params.dryRun) { - return db.countTree(params.source); - } - - const result = await db.moveTree(params.source, params.destination); - return result; -} - -/** - * memory.deleteTree - Delete all memories under a tree path. - */ -async function memoryDeleteTree( - params: MemoryDeleteTreeParams, - context: HandlerContext, -): Promise<{ count: number }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - if (params.dryRun) { - return db.countTree(params.tree); - } - - const result = await db.deleteTree(params.tree); - return result; -} - -/** - * memory.countTree - Count memories under a tree path. - */ -async function memoryCountTree( - params: MemoryCountTreeParams, - context: HandlerContext, -): Promise<{ count: number }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - return db.countTree(params.tree); -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the memory methods registry. - */ -export const memoryMethods = buildRegistry() - .register("memory.create", memoryCreateParams, memoryCreate) - .register("memory.batchCreate", memoryBatchCreateParams, memoryBatchCreate) - .register("memory.get", memoryGetParams, memoryGet) - .register("memory.update", memoryUpdateParams, memoryUpdate) - .register("memory.delete", memoryDeleteParams, memoryDelete) - .register("memory.search", memorySearchParams, memorySearch) - .register("memory.tree", memoryTreeParams, memoryTree) - .register("memory.move", memoryMoveParams, memoryMove) - .register("memory.deleteTree", memoryDeleteTreeParams, memoryDeleteTree) - .register("memory.countTree", memoryCountTreeParams, memoryCountTree) - .build(); diff --git a/packages/server/rpc/engine/owner.test.ts b/packages/server/rpc/engine/owner.test.ts deleted file mode 100644 index cc8c68a..0000000 --- a/packages/server/rpc/engine/owner.test.ts +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Unit tests for owner RPC handlers. - * - * Uses mocked EngineDB to test handler logic in isolation. - */ -import { describe, expect, mock, test } from "bun:test"; -import type { HandlerContext } from "../types"; -import { ownerMethods } from "./owner"; - -function createMockContext( - dbOverrides: Record = {}, -): HandlerContext { - return { - request: new Request("http://localhost"), - db: { - getUserId: mock(() => "user-123"), - setTreeOwner: mock(() => Promise.resolve()), - getTreeOwner: mock(() => Promise.resolve(null)), - removeTreeOwner: mock(() => Promise.resolve(false)), - listTreeOwners: mock(() => Promise.resolve([])), - ...dbOverrides, - }, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; -} - -describe("owner.set", () => { - test("calls setTreeOwner and returns { set: true }", async () => { - const handler = ownerMethods.get("owner.set")?.handler; - if (!handler) throw new Error("owner.set handler not found"); - - const setTreeOwner = mock(() => Promise.resolve()); - const context = createMockContext({ setTreeOwner }); - - const result = await handler( - { - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - }, - context, - ); - - expect(result).toEqual({ set: true }); - expect(setTreeOwner).toHaveBeenCalledTimes(1); - }); -}); - -describe("owner.get", () => { - test("returns owner when found", async () => { - const handler = ownerMethods.get("owner.get")?.handler; - if (!handler) throw new Error("owner.get handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const getTreeOwner = mock(() => - Promise.resolve({ - treePath: "work.projects", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: "user-123", - createdByName: "admin", - createdAt: now, - }), - ); - const context = createMockContext({ getTreeOwner }); - - const result = (await handler({ treePath: "work.projects" }, context)) as { - treePath: string; - userId: string; - userName: string; - createdBy: string; - createdByName: string; - createdAt: string; - }; - - expect(result.treePath).toBe("work.projects"); - expect(result.userId).toBe("019d694f-79f6-7595-8faf-b70b01c11f98"); - expect(result.userName).toBe("alice"); - expect(result.createdBy).toBe("user-123"); - expect(result.createdByName).toBe("admin"); - expect(result.createdAt).toBe("2026-01-15T00:00:00.000Z"); - }); - - test("throws NOT_FOUND when no owner", async () => { - const handler = ownerMethods.get("owner.get")?.handler; - if (!handler) throw new Error("owner.get handler not found"); - - const context = createMockContext({ - getTreeOwner: mock(() => Promise.resolve(null)), - }); - - try { - await handler({ treePath: "work.projects" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -describe("owner.remove", () => { - test("returns { removed: true } when found", async () => { - const handler = ownerMethods.get("owner.remove")?.handler; - if (!handler) throw new Error("owner.remove handler not found"); - - const context = createMockContext({ - removeTreeOwner: mock(() => Promise.resolve(true)), - }); - - const result = await handler({ treePath: "work.projects" }, context); - expect(result).toEqual({ removed: true }); - }); - - test("throws NOT_FOUND when no owner to remove", async () => { - const handler = ownerMethods.get("owner.remove")?.handler; - if (!handler) throw new Error("owner.remove handler not found"); - - const context = createMockContext({ - removeTreeOwner: mock(() => Promise.resolve(false)), - }); - - try { - await handler({ treePath: "work.projects" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -describe("owner.list", () => { - test("returns owners list", async () => { - const handler = ownerMethods.get("owner.list")?.handler; - if (!handler) throw new Error("owner.list handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const listTreeOwners = mock(() => - Promise.resolve([ - { - treePath: "work.projects", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: "user-123", - createdByName: "admin", - createdAt: now, - }, - ]), - ); - const context = createMockContext({ listTreeOwners }); - - const result = (await handler({}, context)) as { - owners: Array<{ treePath: string }>; - }; - - expect(result.owners).toHaveLength(1); - expect(result.owners[0]?.treePath).toBe("work.projects"); - }); - - test("returns empty list when no owners", async () => { - const handler = ownerMethods.get("owner.list")?.handler; - if (!handler) throw new Error("owner.list handler not found"); - - const context = createMockContext({ - listTreeOwners: mock(() => Promise.resolve([])), - }); - - const result = (await handler({}, context)) as { - owners: Array<{ treePath: string }>; - }; - - expect(result.owners).toHaveLength(0); - }); - - test("passes userId filter when provided", async () => { - const handler = ownerMethods.get("owner.list")?.handler; - if (!handler) throw new Error("owner.list handler not found"); - - const listTreeOwners = mock(() => Promise.resolve([])); - const context = createMockContext({ listTreeOwners }); - - await handler({ userId: "019d694f-79f6-7595-8faf-b70b01c11f98" }, context); - - expect(listTreeOwners).toHaveBeenCalledWith( - "019d694f-79f6-7595-8faf-b70b01c11f98", - ); - }); -}); diff --git a/packages/server/rpc/engine/owner.ts b/packages/server/rpc/engine/owner.ts deleted file mode 100644 index 82ef6c1..0000000 --- a/packages/server/rpc/engine/owner.ts +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Engine RPC owner methods. - * - * Implements: - * - owner.set: Set tree path owner - * - owner.get: Get tree path owner - * - owner.remove: Remove tree path owner - * - owner.list: List tree owners - */ -import type { TreeOwner } from "@memory.build/engine"; -import type { - OwnerGetParams, - OwnerListParams, - OwnerRemoveParams, - OwnerResponse, - OwnerSetParams, -} from "@memory.build/protocol/engine/owner"; -import { - ownerGetParams, - ownerListParams, - ownerRemoveParams, - ownerSetParams, -} from "@memory.build/protocol/engine/owner"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a TreeOwner to a serializable response. - */ -function toOwnerResponse(owner: TreeOwner): OwnerResponse { - return { - treePath: owner.treePath, - userId: owner.userId, - userName: owner.userName, - createdBy: owner.createdBy, - createdByName: owner.createdByName, - createdAt: owner.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * owner.set - Set tree path owner (upserts). - */ -async function ownerSet( - params: OwnerSetParams, - context: HandlerContext, -): Promise<{ set: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - await db.setTreeOwner( - params.userId, - params.treePath, - db.getUserId() ?? undefined, - ); - return { set: true }; -} - -/** - * owner.get - Get tree path owner. - */ -async function ownerGet( - params: OwnerGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const owner = await db.getTreeOwner(params.treePath); - if (!owner) { - throw new AppError("NOT_FOUND", `No owner for path: ${params.treePath}`); - } - - return toOwnerResponse(owner); -} - -/** - * owner.remove - Remove tree path owner. - */ -async function ownerRemove( - params: OwnerRemoveParams, - context: HandlerContext, -): Promise<{ removed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const removed = await db.removeTreeOwner(params.treePath); - if (!removed) { - throw new AppError("NOT_FOUND", `No owner for path: ${params.treePath}`); - } - - return { removed }; -} - -/** - * owner.list - List tree owners. - */ -async function ownerList( - params: OwnerListParams, - context: HandlerContext, -): Promise<{ owners: OwnerResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const owners = await db.listTreeOwners(params.userId); - return { owners: owners.map(toOwnerResponse) }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the owner methods registry. - */ -export const ownerMethods = buildRegistry() - .register("owner.set", ownerSetParams, ownerSet) - .register("owner.get", ownerGetParams, ownerGet) - .register("owner.remove", ownerRemoveParams, ownerRemove) - .register("owner.list", ownerListParams, ownerList) - .build(); diff --git a/packages/server/rpc/engine/role.ts b/packages/server/rpc/engine/role.ts deleted file mode 100644 index 1cdbb0f..0000000 --- a/packages/server/rpc/engine/role.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Engine RPC role methods. - * - * Implements: - * - role.create: Create a role (user with canLogin=false) - * - role.addMember: Add a member to a role - * - role.removeMember: Remove a member from a role - * - role.listMembers: List members of a role - * - role.listForUser: List roles a user belongs to - */ -import type { RoleInfo, RoleMember, User } from "@memory.build/engine"; -import type { - RoleAddMemberParams, - RoleCreateParams, - RoleInfoResponse, - RoleListForUserParams, - RoleListMembersParams, - RoleMemberResponse, - RoleRemoveMemberParams, - RoleResponse, -} from "@memory.build/protocol/engine/role"; -import { - roleAddMemberParams, - roleCreateParams, - roleListForUserParams, - roleListMembersParams, - roleRemoveMemberParams, -} from "@memory.build/protocol/engine/role"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a User (role) to a serializable response. - */ -function toRoleResponse(user: User): RoleResponse { - return { - id: user.id, - name: user.name, - identityId: user.identityId, - createdAt: user.createdAt.toISOString(), - updatedAt: user.updatedAt?.toISOString() ?? null, - }; -} - -/** - * Convert a RoleMember to a serializable response. - */ -function toRoleMemberResponse(member: RoleMember): RoleMemberResponse { - return { - roleId: member.roleId, - memberId: member.memberId, - memberName: member.memberName, - withAdminOption: member.withAdminOption, - createdAt: member.createdAt.toISOString(), - }; -} - -/** - * Convert a RoleInfo to a serializable response. - */ -function toRoleInfoResponse(info: RoleInfo): RoleInfoResponse { - return { - id: info.id, - name: info.name, - withAdminOption: info.withAdminOption, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * role.create - Create a role (user with canLogin=false). - */ -async function roleCreate( - params: RoleCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const role = await db.createRole(params.name, params.identityId); - return toRoleResponse(role); -} - -/** - * role.addMember - Add a member to a role. - */ -async function roleAddMember( - params: RoleAddMemberParams, - context: HandlerContext, -): Promise<{ added: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - try { - await db.addRoleMember( - params.roleId, - params.memberId, - params.withAdminOption, - ); - return { added: true }; - } catch (error) { - // Check for cycle error - if ( - error instanceof Error && - error.message.includes("would create a cycle") - ) { - throw new AppError("VALIDATION_ERROR", error.message); - } - throw error; - } -} - -/** - * role.removeMember - Remove a member from a role. - */ -async function roleRemoveMember( - params: RoleRemoveMemberParams, - context: HandlerContext, -): Promise<{ removed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const removed = await db.removeRoleMember(params.roleId, params.memberId); - if (!removed) { - throw new AppError( - "NOT_FOUND", - `Membership not found for role ${params.roleId} and member ${params.memberId}`, - ); - } - - return { removed }; -} - -/** - * role.listMembers - List members of a role. - */ -async function roleListMembers( - params: RoleListMembersParams, - context: HandlerContext, -): Promise<{ members: RoleMemberResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const members = await db.listRoleMembers(params.roleId); - return { members: members.map(toRoleMemberResponse) }; -} - -/** - * role.listForUser - List roles a user belongs to. - */ -async function roleListForUser( - params: RoleListForUserParams, - context: HandlerContext, -): Promise<{ roles: RoleInfoResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const roles = await db.listRolesForUser(params.userId); - return { roles: roles.map(toRoleInfoResponse) }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the role methods registry. - */ -export const roleMethods = buildRegistry() - .register("role.create", roleCreateParams, roleCreate) - .register("role.addMember", roleAddMemberParams, roleAddMember) - .register("role.removeMember", roleRemoveMemberParams, roleRemoveMember) - .register("role.listMembers", roleListMembersParams, roleListMembers) - .register("role.listForUser", roleListForUserParams, roleListForUser) - .build(); diff --git a/packages/server/rpc/engine/schemas.test.ts b/packages/server/rpc/engine/schemas.test.ts deleted file mode 100644 index 5eff0d5..0000000 --- a/packages/server/rpc/engine/schemas.test.ts +++ /dev/null @@ -1,842 +0,0 @@ -/** - * Tests for Engine RPC schemas. - */ -import { describe, expect, test } from "bun:test"; -import { - apiKeyCreateSchema, - apiKeyDeleteSchema, - apiKeyGetSchema, - apiKeyListSchema, - apiKeyRevokeSchema, - grantCheckSchema, - grantCreateSchema, - grantListSchema, - grantRevokeSchema, - memoryBatchCreateSchema, - memoryCountTreeSchema, - memoryCreateSchema, - memoryDeleteSchema, - memoryDeleteTreeSchema, - memoryGetSchema, - memoryMoveSchema, - memorySearchSchema, - memoryTreeSchema, - memoryUpdateSchema, - ownerGetSchema, - ownerListSchema, - ownerRemoveSchema, - ownerSetSchema, - roleAddMemberSchema, - roleCreateSchema, - roleListForUserSchema, - roleListMembersSchema, - roleRemoveMemberSchema, - treePathSchema, - userCreateSchema, - userGetSchema, - userListSchema, - userRenameSchema, - uuidv7Schema, -} from "./schemas"; - -describe("uuidv7Schema", () => { - test("accepts valid UUIDv7", () => { - const validUuids = [ - "019d694f-79f6-7595-8faf-b70b01c11f98", - "019d694f-79f6-7595-9faf-b70b01c11f98", - "019d694f-79f6-7595-afaf-b70b01c11f98", - "019d694f-79f6-7595-bfaf-b70b01c11f98", - ]; - for (const uuid of validUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(true); - } - }); - - test("rejects invalid UUIDs", () => { - const invalidUuids = [ - "not-a-uuid", - "019d694f-79f6-4595-8faf-b70b01c11f98", // v4 not v7 - "019d694f-79f6-7595-0faf-b70b01c11f98", // invalid variant - "019d694f79f675958fafb70b01c11f98", // no dashes - "", - ]; - for (const uuid of invalidUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(false); - } - }); -}); - -describe("treePathSchema", () => { - test("accepts valid tree paths", () => { - const validPaths = [ - "", // root - "work", - "work.projects", - "work.projects.api", - "me_design", - "A1_B2_C3", - ]; - for (const path of validPaths) { - expect(treePathSchema.safeParse(path).success).toBe(true); - } - }); - - test("rejects invalid tree paths", () => { - const invalidPaths = [ - "work.projects.", // trailing dot - ".work.projects", // leading dot - "work..projects", // double dot - "work-projects", // hyphen not allowed - "work projects", // space not allowed - ]; - for (const path of invalidPaths) { - expect(treePathSchema.safeParse(path).success).toBe(false); - } - }); -}); - -describe("memoryCreateSchema", () => { - test("accepts minimal params", () => { - const result = memoryCreateSchema.safeParse({ - content: "Hello world", - }); - expect(result.success).toBe(true); - }); - - test("accepts full params", () => { - const result = memoryCreateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - content: "Hello world", - meta: { type: "note", tags: ["test"] }, - tree: "work.notes", - temporal: { - start: "2024-01-01T00:00:00Z", - end: "2024-01-02T00:00:00Z", - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts point-in-time temporal", () => { - const result = memoryCreateSchema.safeParse({ - content: "Hello world", - temporal: { - start: "2024-01-01T00:00:00Z", - }, - }); - expect(result.success).toBe(true); - }); - - test("rejects empty content", () => { - const result = memoryCreateSchema.safeParse({ - content: "", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid tree path", () => { - const result = memoryCreateSchema.safeParse({ - content: "Hello", - tree: "invalid-path", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryBatchCreateSchema", () => { - test("accepts array of memories", () => { - const result = memoryBatchCreateSchema.safeParse({ - memories: [ - { content: "Memory 1" }, - { content: "Memory 2", tree: "work" }, - ], - }); - expect(result.success).toBe(true); - }); - - test("rejects empty array", () => { - const result = memoryBatchCreateSchema.safeParse({ - memories: [], - }); - expect(result.success).toBe(false); - }); - - test("rejects more than 1000 memories", () => { - const memories = Array(1001) - .fill(null) - .map((_, i) => ({ content: `Memory ${i}` })); - const result = memoryBatchCreateSchema.safeParse({ memories }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryGetSchema", () => { - test("accepts valid UUID", () => { - const result = memoryGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid UUID", () => { - const result = memoryGetSchema.safeParse({ - id: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryUpdateSchema", () => { - test("accepts id with no updates (no-op)", () => { - const result = memoryUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("accepts partial updates", () => { - const result = memoryUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - content: "Updated content", - }); - expect(result.success).toBe(true); - }); - - test("accepts null to clear optional fields", () => { - const result = memoryUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - temporal: null, - }); - expect(result.success).toBe(true); - }); -}); - -describe("memoryDeleteSchema", () => { - test("accepts valid UUID", () => { - const result = memoryDeleteSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("memorySearchSchema", () => { - test("accepts empty params (filter-only)", () => { - const result = memorySearchSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts fulltext search", () => { - const result = memorySearchSchema.safeParse({ - fulltext: "hello world", - }); - expect(result.success).toBe(true); - }); - - test("accepts semantic search", () => { - const result = memorySearchSchema.safeParse({ - semantic: "What is the meaning of life?", - }); - expect(result.success).toBe(true); - }); - - test("accepts hybrid search", () => { - const result = memorySearchSchema.safeParse({ - semantic: "meaning of life", - fulltext: "philosophy", - }); - expect(result.success).toBe(true); - }); - - test("accepts tree filter with lquery pattern", () => { - const result = memorySearchSchema.safeParse({ - tree: "work.*", - }); - expect(result.success).toBe(true); - }); - - test("accepts tree filter with ltxtquery", () => { - const result = memorySearchSchema.safeParse({ - tree: "api & v2", - }); - expect(result.success).toBe(true); - }); - - test("accepts temporal contains filter", () => { - const result = memorySearchSchema.safeParse({ - temporal: { - contains: "2024-01-15T12:00:00Z", - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts temporal overlaps filter", () => { - const result = memorySearchSchema.safeParse({ - temporal: { - overlaps: { - start: "2024-01-01T00:00:00Z", - end: "2024-01-31T23:59:59Z", - }, - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts temporal within filter", () => { - const result = memorySearchSchema.safeParse({ - temporal: { - within: { - start: "2024-01-01T00:00:00Z", - end: "2024-12-31T23:59:59Z", - }, - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts search weights", () => { - const result = memorySearchSchema.safeParse({ - semantic: "test", - fulltext: "test", - weights: { - semantic: 0.7, - fulltext: 0.3, - }, - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid limit", () => { - const result = memorySearchSchema.safeParse({ - limit: 0, - }); - expect(result.success).toBe(false); - }); - - test("rejects limit over 1000", () => { - const result = memorySearchSchema.safeParse({ - limit: 1001, - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid weights", () => { - const result = memorySearchSchema.safeParse({ - weights: { - semantic: 1.5, // > 1 - }, - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryTreeSchema", () => { - test("accepts empty params", () => { - const result = memoryTreeSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts tree path", () => { - const result = memoryTreeSchema.safeParse({ - tree: "work.projects", - }); - expect(result.success).toBe(true); - }); - - test("accepts levels", () => { - const result = memoryTreeSchema.safeParse({ - levels: 3, - }); - expect(result.success).toBe(true); - }); - - test("rejects levels over 100", () => { - const result = memoryTreeSchema.safeParse({ - levels: 101, - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryMoveSchema", () => { - test("accepts valid source and destination", () => { - const result = memoryMoveSchema.safeParse({ - source: "old.path", - destination: "new.path", - }); - expect(result.success).toBe(true); - }); - - test("accepts moving to root", () => { - const result = memoryMoveSchema.safeParse({ - source: "old.path", - destination: "", - }); - expect(result.success).toBe(true); - }); - - test("accepts dryRun flag", () => { - const result = memoryMoveSchema.safeParse({ - source: "old.path", - destination: "new.path", - dryRun: true, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.dryRun).toBe(true); - } - }); - - test("rejects empty source", () => { - const result = memoryMoveSchema.safeParse({ - source: "", - destination: "new.path", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryDeleteTreeSchema", () => { - test("accepts valid tree path", () => { - const result = memoryDeleteTreeSchema.safeParse({ - tree: "old.stuff", - }); - expect(result.success).toBe(true); - }); - - test("accepts dryRun flag", () => { - const result = memoryDeleteTreeSchema.safeParse({ - tree: "old.stuff", - dryRun: true, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.dryRun).toBe(true); - } - }); - - test("rejects empty tree path", () => { - const result = memoryDeleteTreeSchema.safeParse({ - tree: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryCountTreeSchema", () => { - test("accepts valid tree path", () => { - const result = memoryCountTreeSchema.safeParse({ - tree: "old.stuff", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty tree path", () => { - const result = memoryCountTreeSchema.safeParse({ - tree: "", - }); - expect(result.success).toBe(false); - }); - - test("rejects missing tree path", () => { - const result = memoryCountTreeSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// User Schema Tests -// ============================================================================= - -describe("userCreateSchema", () => { - test("accepts minimal params", () => { - const result = userCreateSchema.safeParse({ - name: "alice", - }); - expect(result.success).toBe(true); - }); - - test("accepts full params", () => { - const result = userCreateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "alice", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - canLogin: true, - superuser: false, - createrole: true, - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = userCreateSchema.safeParse({ - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("userGetSchema", () => { - test("accepts valid UUID", () => { - const result = userGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("userListSchema", () => { - test("accepts empty params", () => { - const result = userListSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts canLogin filter", () => { - const result = userListSchema.safeParse({ - canLogin: false, - }); - expect(result.success).toBe(true); - }); -}); - -describe("userRenameSchema", () => { - test("accepts valid params", () => { - const result = userRenameSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "new-name", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = userRenameSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Grant Schema Tests -// ============================================================================= - -describe("grantCreateSchema", () => { - test("accepts valid params", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - actions: ["read", "create"], - }); - expect(result.success).toBe(true); - }); - - test("accepts with grant option", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - actions: ["update"], - withGrantOption: true, - }); - expect(result.success).toBe(true); - }); - - test("rejects empty actions", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - actions: [], - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid action", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - actions: ["read", "invalid"], - }); - expect(result.success).toBe(false); - }); -}); - -describe("grantListSchema", () => { - test("accepts empty params", () => { - const result = grantListSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts userId filter", () => { - const result = grantListSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("grantRevokeSchema", () => { - test("accepts valid params", () => { - const result = grantRevokeSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - }); - expect(result.success).toBe(true); - }); -}); - -describe("grantCheckSchema", () => { - test("accepts valid params", () => { - const result = grantCheckSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects.api", - action: "read", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid action", () => { - const result = grantCheckSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - action: "execute", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Role Schema Tests -// ============================================================================= - -describe("roleCreateSchema", () => { - test("accepts minimal params", () => { - const result = roleCreateSchema.safeParse({ - name: "editors", - }); - expect(result.success).toBe(true); - }); - - test("accepts with identityId", () => { - const result = roleCreateSchema.safeParse({ - name: "editors", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = roleCreateSchema.safeParse({ - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("roleAddMemberSchema", () => { - test("accepts valid params", () => { - const result = roleAddMemberSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - memberId: "019d694f-79f6-7595-8faf-b70b01c11f99", - }); - expect(result.success).toBe(true); - }); - - test("accepts with admin option", () => { - const result = roleAddMemberSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - memberId: "019d694f-79f6-7595-8faf-b70b01c11f99", - withAdminOption: true, - }); - expect(result.success).toBe(true); - }); -}); - -describe("roleRemoveMemberSchema", () => { - test("accepts valid params", () => { - const result = roleRemoveMemberSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - memberId: "019d694f-79f6-7595-8faf-b70b01c11f99", - }); - expect(result.success).toBe(true); - }); -}); - -describe("roleListMembersSchema", () => { - test("accepts valid params", () => { - const result = roleListMembersSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("roleListForUserSchema", () => { - test("accepts valid params", () => { - const result = roleListForUserSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// API Key Schema Tests -// ============================================================================= - -describe("apiKeyCreateSchema", () => { - test("accepts minimal params", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "my-api-key", - }); - expect(result.success).toBe(true); - }); - - test("accepts with expiration", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "my-api-key", - expiresAt: "2025-12-31T23:59:59Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid expiration timestamp", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "my-api-key", - expiresAt: "not-a-timestamp", - }); - expect(result.success).toBe(false); - }); -}); - -describe("apiKeyGetSchema", () => { - test("accepts valid UUID", () => { - const result = apiKeyGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("apiKeyListSchema", () => { - test("accepts valid userId", () => { - const result = apiKeyListSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("apiKeyRevokeSchema", () => { - test("accepts valid UUID", () => { - const result = apiKeyRevokeSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("apiKeyDeleteSchema", () => { - test("accepts valid UUID", () => { - const result = apiKeyDeleteSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Owner Schema Tests -// ============================================================================= - -describe("ownerSetSchema", () => { - test("accepts valid params", () => { - const result = ownerSetSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing userId", () => { - const result = ownerSetSchema.safeParse({ - treePath: "work.projects", - }); - expect(result.success).toBe(false); - }); - - test("rejects missing treePath", () => { - const result = ownerSetSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid UUID", () => { - const result = ownerSetSchema.safeParse({ - userId: "not-a-uuid", - treePath: "work.projects", - }); - expect(result.success).toBe(false); - }); -}); - -describe("ownerGetSchema", () => { - test("accepts valid treePath", () => { - const result = ownerGetSchema.safeParse({ - treePath: "work.projects.alpha", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing treePath", () => { - const result = ownerGetSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -describe("ownerRemoveSchema", () => { - test("accepts valid treePath", () => { - const result = ownerRemoveSchema.safeParse({ - treePath: "work.projects", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing treePath", () => { - const result = ownerRemoveSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -describe("ownerListSchema", () => { - test("accepts with userId", () => { - const result = ownerListSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("accepts without userId", () => { - const result = ownerListSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("rejects invalid userId", () => { - const result = ownerListSchema.safeParse({ - userId: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/server/rpc/engine/schemas.ts b/packages/server/rpc/engine/schemas.ts deleted file mode 100644 index acf5dbd..0000000 --- a/packages/server/rpc/engine/schemas.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Re-export engine schemas from @memory.build/protocol. - * - * @deprecated Import directly from @memory.build/protocol/engine instead. - */ - -export { - type ApiKeyCreateParams, - type ApiKeyDeleteParams, - type ApiKeyGetParams, - type ApiKeyListParams, - type ApiKeyRevokeParams, - // API Key params - apiKeyCreateParams as apiKeyCreateSchema, - apiKeyDeleteParams as apiKeyDeleteSchema, - apiKeyGetParams as apiKeyGetSchema, - apiKeyListParams as apiKeyListSchema, - apiKeyRevokeParams as apiKeyRevokeSchema, -} from "@memory.build/protocol/engine/api-key"; -export { - type GrantCheckParams, - type GrantCreateParams, - type GrantGetParams, - type GrantListParams, - type GrantRevokeParams, - grantCheckParams as grantCheckSchema, - // Grant params - grantCreateParams as grantCreateSchema, - grantGetParams as grantGetSchema, - grantListParams as grantListSchema, - grantRevokeParams as grantRevokeSchema, -} from "@memory.build/protocol/engine/grant"; -export { - type MemoryBatchCreateParams, - type MemoryCountTreeParams, - type MemoryCreateParams, - type MemoryDeleteParams, - type MemoryDeleteTreeParams, - type MemoryGetParams, - type MemoryMoveParams, - type MemorySearchParams, - type MemoryTreeParams, - type MemoryUpdateParams, - memoryBatchCreateParams as memoryBatchCreateSchema, - memoryCountTreeParams as memoryCountTreeSchema, - // Memory params - memoryCreateParams as memoryCreateSchema, - memoryDeleteParams as memoryDeleteSchema, - memoryDeleteTreeParams as memoryDeleteTreeSchema, - memoryGetParams as memoryGetSchema, - memoryMoveParams as memoryMoveSchema, - memorySearchParams as memorySearchSchema, - memoryTreeParams as memoryTreeSchema, - memoryUpdateParams as memoryUpdateSchema, -} from "@memory.build/protocol/engine/memory"; -export { - type OwnerGetParams, - type OwnerListParams, - type OwnerRemoveParams, - type OwnerSetParams, - ownerGetParams as ownerGetSchema, - // Owner params - ownerListParams as ownerListSchema, - ownerRemoveParams as ownerRemoveSchema, - ownerSetParams as ownerSetSchema, -} from "@memory.build/protocol/engine/owner"; -export { - type RoleAddMemberParams, - type RoleCreateParams, - type RoleListForUserParams, - type RoleListMembersParams, - type RoleRemoveMemberParams, - roleAddMemberParams as roleAddMemberSchema, - // Role params - roleCreateParams as roleCreateSchema, - roleListForUserParams as roleListForUserSchema, - roleListMembersParams as roleListMembersSchema, - roleRemoveMemberParams as roleRemoveMemberSchema, -} from "@memory.build/protocol/engine/role"; -export { - type UserCreateParams, - type UserDeleteParams, - type UserGetByNameParams, - type UserGetParams, - type UserListParams, - type UserRenameParams, - // User params - userCreateParams as userCreateSchema, - userDeleteParams as userDeleteSchema, - userGetByNameParams as userGetByNameSchema, - userGetParams as userGetSchema, - userListParams as userListSchema, - userRenameParams as userRenameSchema, -} from "@memory.build/protocol/engine/user"; -export { - // Fields - grantActionSchema, - metaSchema, - searchWeightsSchema, - temporalFilterSchema, - temporalSchema, - timestampSchema, - treeFilterSchema, - treePathSchema, - uuidv7Schema, -} from "@memory.build/protocol/fields"; diff --git a/packages/server/rpc/engine/types.ts b/packages/server/rpc/engine/types.ts deleted file mode 100644 index e316384..0000000 --- a/packages/server/rpc/engine/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Engine RPC context types. - * - * Extends the base HandlerContext with engine-specific fields. - */ -import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { EngineDB } from "@memory.build/engine"; -import type { EngineInfo } from "../../middleware/authenticate"; -import type { HandlerContext } from "../types"; - -/** - * Engine handler context. - * - * Provides access to: - * - `db`: EngineDB instance for the authenticated engine - * - `userId`: The authenticated user's ID (from API key) - * - `apiKeyId`: The API key ID used for authentication - * - `engine`: Engine metadata from accounts DB - * - `embeddingConfig`: Optional config for semantic search - */ -export interface EngineContext extends HandlerContext { - /** EngineDB instance for this engine */ - db: EngineDB; - /** Authenticated user ID */ - userId: string; - /** API key ID used for authentication */ - apiKeyId: string; - /** Engine metadata from accounts DB */ - engine: EngineInfo; - /** Embedding config for semantic search (optional) */ - embeddingConfig?: EmbeddingConfig; -} - -/** - * Type guard to check if context has engine fields. - */ -export function isEngineContext(ctx: HandlerContext): ctx is EngineContext { - return ( - "db" in ctx && - typeof ctx.db === "object" && - ctx.db !== null && - "userId" in ctx && - typeof ctx.userId === "string" && - "apiKeyId" in ctx && - typeof ctx.apiKeyId === "string" && - "engine" in ctx && - typeof ctx.engine === "object" && - ctx.engine !== null - // embeddingConfig is optional, don't check - ); -} - -/** - * Assert that context is an EngineContext, throwing if not. - */ -export function assertEngineContext( - ctx: HandlerContext, -): asserts ctx is EngineContext { - if (!isEngineContext(ctx)) { - throw new Error("Engine context not initialized (authentication required)"); - } -} diff --git a/packages/server/rpc/engine/user.ts b/packages/server/rpc/engine/user.ts deleted file mode 100644 index 5579245..0000000 --- a/packages/server/rpc/engine/user.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Engine RPC user methods. - * - * Implements: - * - user.create: Create a new user - * - user.get: Get user by ID - * - user.getByName: Get user by name - * - user.list: List users (optionally filter by canLogin) - * - user.rename: Rename a user - * - user.delete: Delete a user - */ -import type { User } from "@memory.build/engine"; -import type { - UserCreateParams, - UserDeleteParams, - UserGetByNameParams, - UserGetParams, - UserListParams, - UserRenameParams, - UserResponse, -} from "@memory.build/protocol/engine/user"; -import { - userCreateParams, - userDeleteParams, - userGetByNameParams, - userGetParams, - userListParams, - userRenameParams, -} from "@memory.build/protocol/engine/user"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a User to a serializable response. - */ -function toUserResponse(user: User): UserResponse { - return { - id: user.id, - name: user.name, - identityId: user.identityId, - canLogin: user.canLogin, - superuser: user.superuser, - createrole: user.createrole, - createdAt: user.createdAt.toISOString(), - updatedAt: user.updatedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * user.create - Create a new user. - */ -async function userCreate( - params: UserCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const user = await db.createUser({ - id: params.id ?? undefined, - name: params.name, - identityId: params.identityId ?? undefined, - canLogin: params.canLogin, - superuser: params.superuser, - createrole: params.createrole, - }); - - return toUserResponse(user); -} - -/** - * user.get - Get user by ID. - */ -async function userGet( - params: UserGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const user = await db.getUser(params.id); - if (!user) { - throw new AppError("NOT_FOUND", `User not found: ${params.id}`); - } - - return toUserResponse(user); -} - -/** - * user.getByName - Get user by name. - */ -async function userGetByName( - params: UserGetByNameParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const user = await db.getUserByName(params.name); - if (!user) { - throw new AppError("NOT_FOUND", `User not found: ${params.name}`); - } - - return toUserResponse(user); -} - -/** - * user.list - List users. - */ -async function userList( - params: UserListParams, - context: HandlerContext, -): Promise<{ users: UserResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const users = await db.listUsers(params.canLogin); - return { users: users.map(toUserResponse) }; -} - -/** - * user.rename - Rename a user. - */ -async function userRename( - params: UserRenameParams, - context: HandlerContext, -): Promise<{ renamed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const renamed = await db.renameUser(params.id, params.name); - if (!renamed) { - throw new AppError("NOT_FOUND", `User not found: ${params.id}`); - } - - return { renamed }; -} - -/** - * user.delete - Delete a user. - */ -async function userDelete( - params: UserDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const deleted = await db.deleteUser(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `User not found: ${params.id}`); - } - - return { deleted }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the user methods registry. - */ -export const userMethods = buildRegistry() - .register("user.create", userCreateParams, userCreate) - .register("user.get", userGetParams, userGet) - .register("user.getByName", userGetByNameParams, userGetByName) - .register("user.list", userListParams, userList) - .register("user.rename", userRenameParams, userRename) - .register("user.delete", userDeleteParams, userDelete) - .build(); diff --git a/packages/server/rpc/index.ts b/packages/server/rpc/index.ts index c2fc516..bc7e876 100644 --- a/packages/server/rpc/index.ts +++ b/packages/server/rpc/index.ts @@ -1,11 +1,3 @@ -// Method registries -export { accountsMethods } from "./accounts"; -export { - assertEngineContext, - type EngineContext, - engineMethods, - isEngineContext, -} from "./engine"; // Errors export { APP_ERROR_CODES, diff --git a/packages/server/server.integration.test.ts b/packages/server/server.integration.test.ts index a18c29a..9cb7e73 100644 --- a/packages/server/server.integration.test.ts +++ b/packages/server/server.integration.test.ts @@ -1,9 +1,7 @@ import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { CoreStore } from "@memory.build/engine/core"; -import type { SQL } from "bun"; import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; @@ -16,15 +14,9 @@ let baseUrl: string; // Mock ServerContext for testing function createMockContext(): ServerContext { return { - accountsDb: { - validateSession: mock(() => Promise.resolve(null)), - getEngineBySlug: mock(() => Promise.resolve(null)), - } as unknown as AccountsDB, - accountsSql: {} as SQL, - engineSql: {} as SQL, db: {} as Sql, auth: { - // Session validation: no session → accounts RPC stays 401. + // Session validation: no session → user RPC stays 401. validateSession: mock(() => Promise.resolve(null)), // Device flow operations exercised by the auth endpoint tests. createDeviceAuth: mock((_provider: string) => @@ -129,7 +121,7 @@ describe("server integration", () => { test("rejects requests with oversized Content-Length header", async () => { // Create a request with a misleading Content-Length header // In real scenarios, the header would match the actual body - const request = new Request(`${baseUrl}/api/v1/accounts/rpc`, { + const request = new Request(`${baseUrl}/api/v1/memory/rpc`, { method: "POST", headers: { "Content-Length": String(MAX_BODY_SIZE + 1), @@ -145,12 +137,12 @@ describe("server integration", () => { }); test("allows normal sized requests", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`, { + const response = await fetch(`${baseUrl}/api/v1/user/rpc`, { method: "POST", headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ jsonrpc: "2.0", method: "test", id: 1 }), + body: JSON.stringify({ jsonrpc: "2.0", method: "whoami", id: 1 }), }); // Should get 401 (auth required) not 413 (size limit) // Auth is checked after size limit passes @@ -159,37 +151,38 @@ describe("server integration", () => { }); describe("RPC endpoints", () => { - test("POST /api/v1/accounts/rpc returns 401 without auth", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`, { + test("POST /api/v1/memory/rpc returns 401 without auth", async () => { + const response = await fetch(`${baseUrl}/api/v1/memory/rpc`, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + "X-Me-Space": "abc123def456", + }, body: JSON.stringify({}), }); // Auth is required before JSON-RPC processing expect(response.status).toBe(401); }); - test("POST /api/v1/accounts/rpc returns 401 for unauthenticated requests", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`, { + test("POST /api/v1/memory/rpc returns 400 when X-Me-Space is missing", async () => { + const response = await fetch(`${baseUrl}/api/v1/memory/rpc`, { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - jsonrpc: "2.0", - method: "unknown.method", - id: 1, - }), + headers: { + "Content-Type": "application/json", + Authorization: "Bearer some-token", + }, + body: JSON.stringify({ jsonrpc: "2.0", method: "memory.get", id: 1 }), }); - // Auth is required before method lookup - expect(response.status).toBe(401); + expect(response.status).toBe(400); }); - test("POST /api/v1/engine/rpc returns 401 without auth", async () => { - const response = await fetch(`${baseUrl}/api/v1/engine/rpc`, { + test("POST /api/v1/user/rpc returns 401 for unauthenticated requests", async () => { + const response = await fetch(`${baseUrl}/api/v1/user/rpc`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", - method: "unknown.method", + method: "whoami", id: 1, }), }); @@ -197,8 +190,8 @@ describe("server integration", () => { expect(response.status).toBe(401); }); - test("GET /api/v1/accounts/rpc returns 404 (wrong method)", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`); + test("GET /api/v1/memory/rpc returns 404 (wrong method)", async () => { + const response = await fetch(`${baseUrl}/api/v1/memory/rpc`); expect(response.status).toBe(404); }); }); diff --git a/packages/server/wiring.test.ts b/packages/server/wiring.test.ts index 812baa4..8d2c76c 100644 --- a/packages/server/wiring.test.ts +++ b/packages/server/wiring.test.ts @@ -1,38 +1,27 @@ /** * Unit tests for server-database wiring. * - * These tests verify that authentication middleware is correctly wired - * to the router using mocked database connections. They test the wiring - * logic, not actual database operations. + * These verify that the new-model authentication middleware is correctly wired + * to the router using mocked stores. They test the wiring (which authenticator + * guards which route, and the shape of its rejections), not real DB operations. * - * For true integration tests with a real database, see the e2e test suite. + * For true integration tests with a real database, see the *.integration.test.ts + * suites under rpc/memory and rpc/user. */ import { describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { CoreStore } from "@memory.build/engine/core"; -import type { SQL } from "bun"; import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; -import type { EngineInfo } from "./middleware/authenticate"; import { createRouter } from "./router"; // ============================================================================= // Test Helpers // ============================================================================= -function createMockAccountsDb(overrides?: { - getEngineBySlug?: ReturnType; -}): AccountsDB { - return { - getEngineBySlug: - overrides?.getEngineBySlug ?? mock(() => Promise.resolve(null)), - } as unknown as AccountsDB; -} - function createMockAuth(overrides?: { validateSession?: ReturnType; getUser?: ReturnType; @@ -44,35 +33,8 @@ function createMockAuth(overrides?: { } as unknown as AuthStore; } -/** - * Create a mock SQL that has enough methods to not throw, but returns - * no results. This allows testing wiring without a real database. - */ -function createMockEngineSql(): SQL { - // Create a function that's also a template tag, returning empty results - const mockSqlTag = mock(() => Promise.resolve([])); - // Add the unsafe method for schema/identifier interpolation - (mockSqlTag as unknown as { unsafe: ReturnType }).unsafe = mock( - (str: string) => str, - ); - - const mockTx = Object.assign( - mock(() => Promise.resolve([])), - { - unsafe: mock((str: string) => str), - }, - ); - - return { - begin: mock((fn: (tx: unknown) => Promise) => fn(mockTx)), - } as unknown as SQL; -} - function createMockContext(overrides?: Partial): ServerContext { return { - accountsDb: createMockAccountsDb(), - accountsSql: {} as SQL, - engineSql: createMockEngineSql(), db: {} as Sql, auth: createMockAuth(), core: {} as unknown as CoreStore, @@ -96,14 +58,15 @@ function createMockContext(overrides?: Partial): ServerContext { // ============================================================================= describe("Server-Database Wiring", () => { - describe("Engine RPC Authentication", () => { + describe("Memory RPC authentication (authenticateSpace)", () => { test("returns 401 for missing Authorization header", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/engine/rpc", { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + "X-Me-Space": "abc123def456", + }, body: JSON.stringify({ jsonrpc: "2.0", method: "memory.get", @@ -116,15 +79,13 @@ describe("Server-Database Wiring", () => { expect(response.status).toBe(401); }); - test("returns 401 for invalid API key format", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/engine/rpc", { + test("returns 400 when the X-Me-Space header is missing", async () => { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers: { "Content-Type": "application/json", - Authorization: "Bearer invalid-key", + Authorization: "Bearer some-token", }, body: JSON.stringify({ jsonrpc: "2.0", @@ -135,21 +96,21 @@ describe("Server-Database Wiring", () => { }); const response = await router.handleRequest(request); - expect(response.status).toBe(401); + expect(response.status).toBe(400); + const body = (await response.json()) as { error: { code: string } }; + expect(body.error.code).toBe("MISSING_SPACE"); }); }); - describe("Accounts RPC Authentication", () => { + describe("User RPC authentication (authenticateUser)", () => { test("returns 401 for missing Authorization header", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/accounts/rpc", { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/user/rpc", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", - method: "me.get", + method: "whoami", params: {}, id: 1, }), @@ -159,11 +120,9 @@ describe("Server-Database Wiring", () => { expect(response.status).toBe(401); }); - test("returns 401 for invalid session token", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/accounts/rpc", { + test("returns 401 for an invalid session token", async () => { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/user/rpc", { method: "POST", headers: { "Content-Type": "application/json", @@ -171,7 +130,7 @@ describe("Server-Database Wiring", () => { }, body: JSON.stringify({ jsonrpc: "2.0", - method: "me.get", + method: "whoami", params: {}, id: 1, }), @@ -181,29 +140,28 @@ describe("Server-Database Wiring", () => { expect(response.status).toBe(401); }); - test("succeeds with valid session token (happy path)", async () => { - const mockIdentity = { - id: "identity-123", + test("whoami succeeds with a valid session (happy path)", async () => { + const identity = { + id: "01960000-0000-7000-8000-000000000000", email: "test@example.com", name: "Test User", }; - const ctx = createMockContext({ auth: createMockAuth({ validateSession: mock(() => Promise.resolve({ sessionId: "session-1", - userId: mockIdentity.id, - email: mockIdentity.email, - name: mockIdentity.name, + userId: identity.id, + email: identity.email, + name: identity.name, expiresAt: new Date("2026-12-31T00:00:00Z"), }), ), getUser: mock(() => Promise.resolve({ - id: mockIdentity.id, - email: mockIdentity.email, - name: mockIdentity.name, + id: identity.id, + email: identity.email, + name: identity.name, emailVerified: true, createdAt: new Date("2026-01-01T00:00:00Z"), updatedAt: null, @@ -213,7 +171,7 @@ describe("Server-Database Wiring", () => { }); const router = createRouter(ctx); - const request = new Request("http://localhost/api/v1/accounts/rpc", { + const request = new Request("http://localhost/api/v1/user/rpc", { method: "POST", headers: { "Content-Type": "application/json", @@ -221,82 +179,29 @@ describe("Server-Database Wiring", () => { }, body: JSON.stringify({ jsonrpc: "2.0", - method: "me.get", + method: "whoami", params: {}, id: 1, }), }); const response = await router.handleRequest(request); - // Should get 200 with RPC response (method may not exist but auth passed) expect(response.status).toBe(200); - const body = await response.json(); - // If method doesn't exist, we get an RPC error, but auth succeeded - expect(body).toHaveProperty("jsonrpc", "2.0"); + const body = (await response.json()) as { + jsonrpc: string; + result: { id: string; email: string; name: string }; + }; + expect(body.jsonrpc).toBe("2.0"); + expect(body.result).toEqual(identity); }); }); describe("Health endpoint (no auth)", () => { test("returns 200 without authentication", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - + const router = createRouter(createMockContext()); const request = new Request("http://localhost/health"); - const response = await router.handleRequest(request); expect(response.status).toBe(200); }); }); - - describe("Engine RPC wiring verification", () => { - test("engine lookup is called with slug from API key", async () => { - const mockEngine: EngineInfo = { - id: "engine-123", - orgId: "org-456", - slug: "abc123xyz789", - name: "Test Engine", - shardId: 1, - status: "active", - }; - - // Verify router correctly extracts slug from API key and calls accountsDb - // The full auth flow will fail because engineSql is a mock, but we verify - // the wiring is correct by checking getEngineBySlug was called with the right slug - const getEngineBySlug = mock(() => Promise.resolve(mockEngine)); - const ctx = createMockContext({ - accountsDb: createMockAccountsDb({ getEngineBySlug }), - }); - const router = createRouter(ctx); - - // Valid API key format: me.{slug}.{lookupId}.{secret} - const validApiKey = - "me.abc123xyz789.Sh00uLs5rmSHHun3.pREy3xfnbCpgUXiaBcDefghij1234567"; - - const request = new Request("http://localhost/api/v1/engine/rpc", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${validApiKey}`, - }, - body: JSON.stringify({ - jsonrpc: "2.0", - method: "memory.get", - params: { id: "test-id" }, - id: 1, - }), - }); - - // Request will fail downstream (mock engineSql lacks required methods), - // but the wiring we're testing happens before that failure - const response = await router.handleRequest(request); - - // The response will be an error (500 or similar) because the mock SQL - // doesn't have required methods, but we're testing the wiring, not the - // full flow. The important thing is getEngineBySlug was called. - expect(response.status).toBeGreaterThanOrEqual(400); - - // Verify the engine lookup was called with the correct slug extracted from API key - expect(getEngineBySlug).toHaveBeenCalledWith("abc123xyz789"); - }); - }); }); From d644d225f885d5ccc62e3b3bf5f1560b0a8ba9cf Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 21:27:15 +0200 Subject: [PATCH 061/156] feat: delete the accounts package (5B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packages/accounts is unimported after 5A — remove the package (org/member/ engine/invitation ops, the legacy auth oauth/session ops, and the `accounts` schema migrations). Drop the @memory.build/accounts dependency from the server, the accounts entry from release-server.ts, and swap the accounts path filter in deploy-dev.yaml for auth + database (the server's actual new deps). Lockfile refreshed. Removed concepts (org/member/invitation) have no replacement by design; the oauth/session behavior is covered by packages/auth and its tests. typecheck + lint + unit (822) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy-dev.yaml | 3 +- bun.lock | 10 - packages/accounts/db.integration.test.ts | 590 ------------------ packages/accounts/db.ts | 83 --- packages/accounts/index.ts | 4 - packages/accounts/migrate/index.ts | 4 - .../migrate/migrate.integration.test.ts | 270 -------- .../migrate/migrations/001_updated_at.sql | 13 - .../migrate/migrations/002_core_tables.sql | 58 -- .../migrate/migrations/003_membership.sql | 10 - .../migrate/migrations/004_invitations.sql | 17 - .../accounts/migrate/migrations/005_auth.sql | 34 - .../migrate/migrations/006_ops_support.sql | 46 -- .../migrations/007_device_authorization.sql | 14 - .../migrations/008_drop_org_owner_trigger.sql | 13 - .../migrate/migrations/009_session_lookup.sql | 30 - packages/accounts/migrate/migrations/sql.d.ts | 4 - packages/accounts/migrate/runner.test.ts | 23 - packages/accounts/migrate/runner.ts | 241 ------- packages/accounts/migrate/template.test.ts | 51 -- packages/accounts/migrate/template.ts | 22 - packages/accounts/migrate/test-utils.ts | 82 --- packages/accounts/ops/_tx.ts | 78 --- packages/accounts/ops/device-auth.ts | 207 ------ packages/accounts/ops/engine.ts | 143 ----- packages/accounts/ops/identity.ts | 99 --- packages/accounts/ops/index.ts | 9 - packages/accounts/ops/invitation.ts | 134 ---- packages/accounts/ops/oauth.ts | 101 --- packages/accounts/ops/org-member.ts | 198 ------ packages/accounts/ops/org.ts | 110 ---- packages/accounts/ops/session.ts | 132 ---- packages/accounts/package.json | 9 - packages/accounts/types.ts | 213 ------- packages/accounts/util/hash.test.ts | 49 -- packages/accounts/util/hash.ts | 27 - packages/accounts/util/slug.ts | 20 - packages/server/package.json | 1 - scripts/release-server.ts | 3 +- 39 files changed, 3 insertions(+), 3152 deletions(-) delete mode 100644 packages/accounts/db.integration.test.ts delete mode 100644 packages/accounts/db.ts delete mode 100644 packages/accounts/index.ts delete mode 100644 packages/accounts/migrate/index.ts delete mode 100644 packages/accounts/migrate/migrate.integration.test.ts delete mode 100644 packages/accounts/migrate/migrations/001_updated_at.sql delete mode 100644 packages/accounts/migrate/migrations/002_core_tables.sql delete mode 100644 packages/accounts/migrate/migrations/003_membership.sql delete mode 100644 packages/accounts/migrate/migrations/004_invitations.sql delete mode 100644 packages/accounts/migrate/migrations/005_auth.sql delete mode 100644 packages/accounts/migrate/migrations/006_ops_support.sql delete mode 100644 packages/accounts/migrate/migrations/007_device_authorization.sql delete mode 100644 packages/accounts/migrate/migrations/008_drop_org_owner_trigger.sql delete mode 100644 packages/accounts/migrate/migrations/009_session_lookup.sql delete mode 100644 packages/accounts/migrate/migrations/sql.d.ts delete mode 100644 packages/accounts/migrate/runner.test.ts delete mode 100644 packages/accounts/migrate/runner.ts delete mode 100644 packages/accounts/migrate/template.test.ts delete mode 100644 packages/accounts/migrate/template.ts delete mode 100644 packages/accounts/migrate/test-utils.ts delete mode 100644 packages/accounts/ops/_tx.ts delete mode 100644 packages/accounts/ops/device-auth.ts delete mode 100644 packages/accounts/ops/engine.ts delete mode 100644 packages/accounts/ops/identity.ts delete mode 100644 packages/accounts/ops/index.ts delete mode 100644 packages/accounts/ops/invitation.ts delete mode 100644 packages/accounts/ops/oauth.ts delete mode 100644 packages/accounts/ops/org-member.ts delete mode 100644 packages/accounts/ops/org.ts delete mode 100644 packages/accounts/ops/session.ts delete mode 100644 packages/accounts/package.json delete mode 100644 packages/accounts/types.ts delete mode 100644 packages/accounts/util/hash.test.ts delete mode 100644 packages/accounts/util/hash.ts delete mode 100644 packages/accounts/util/slug.ts diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml index 4e07219..ee905d7 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-dev.yaml @@ -9,7 +9,8 @@ on: - packages/embedding/** - packages/worker/** - packages/protocol/** - - packages/accounts/** + - packages/auth/** + - packages/database/** - scripts/** - package.json - bun.lock diff --git a/bun.lock b/bun.lock index 9b87aca..0291073 100644 --- a/bun.lock +++ b/bun.lock @@ -12,13 +12,6 @@ "typescript": "^6.0.2", }, }, - "packages/accounts": { - "name": "@memory.build/accounts", - "version": "0.2.5", - "dependencies": { - "@pydantic/logfire-node": "^0.13.1", - }, - }, "packages/auth": { "name": "@memory.build/auth", "version": "0.0.0", @@ -132,7 +125,6 @@ "name": "memory-engine-server", "version": "0.2.5", "dependencies": { - "@memory.build/accounts": "workspace:*", "@memory.build/auth": "workspace:*", "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", @@ -387,8 +379,6 @@ "@js-sdsl/ordered-map": ["@js-sdsl/ordered-map@4.4.2", "", {}, "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="], - "@memory.build/accounts": ["@memory.build/accounts@workspace:packages/accounts"], - "@memory.build/auth": ["@memory.build/auth@workspace:packages/auth"], "@memory.build/cli": ["@memory.build/cli@workspace:packages/cli"], diff --git a/packages/accounts/db.integration.test.ts b/packages/accounts/db.integration.test.ts deleted file mode 100644 index e420800..0000000 --- a/packages/accounts/db.integration.test.ts +++ /dev/null @@ -1,590 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { type AccountsDB, createAccountsDB } from "./db"; -import { TestDatabase } from "./migrate/test-utils"; -import { AccountsError } from "./types"; - -let testDb: TestDatabase; -let db: AccountsDB; - -beforeAll(async () => { - testDb = await TestDatabase.create(); - - db = createAccountsDB(testDb.sql, testDb.schema); -}); - -afterAll(async () => { - await testDb.dispose(); -}); - -// --------------------------------------------------------------------------- -// Identity tests -// --------------------------------------------------------------------------- - -describe("identity", () => { - test("create and get identity", async () => { - const identity = await db.createIdentity({ - email: "test@example.com", - name: "Test User", - }); - - expect(identity.id).toBeDefined(); - expect(identity.email).toBe("test@example.com"); - expect(identity.name).toBe("Test User"); - - const fetched = await db.getIdentity(identity.id); - expect(fetched).toEqual(identity); - }); - - test("get identity by email", async () => { - const identity = await db.createIdentity({ - email: "byemail@example.com", - name: "By Email", - }); - - const fetched = await db.getIdentityByEmail("byemail@example.com"); - expect(fetched?.id).toBe(identity.id); - }); - - test("update identity", async () => { - const identity = await db.createIdentity({ - email: "update@example.com", - name: "Original Name", - }); - - const updated = await db.updateIdentity(identity.id, { name: "New Name" }); - expect(updated).toBe(true); - - const fetched = await db.getIdentity(identity.id); - expect(fetched?.name).toBe("New Name"); - }); - - test("delete identity", async () => { - const identity = await db.createIdentity({ - email: "delete@example.com", - name: "To Delete", - }); - - const deleted = await db.deleteIdentity(identity.id); - expect(deleted).toBe(true); - - const fetched = await db.getIdentity(identity.id); - expect(fetched).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// Org tests -// --------------------------------------------------------------------------- - -describe("org", () => { - test("create and get org", async () => { - const org = await db.createOrg({ - name: "Test Organization", - }); - - expect(org.id).toBeDefined(); - expect(org.slug).toMatch(/^[a-z0-9]{12}$/); - expect(org.name).toBe("Test Organization"); - - const fetched = await db.getOrg(org.id); - expect(fetched).toEqual(org); - }); - - test("get org by slug", async () => { - const org = await db.createOrg({ - name: "By Slug", - }); - - const fetched = await db.getOrgBySlug(org.slug); - expect(fetched?.id).toBe(org.id); - }); - - test("update org name", async () => { - const org = await db.createOrg({ name: "Original Name" }); - - const updated = await db.updateOrg(org.id, { name: "Renamed" }); - expect(updated).toBe(true); - - const fetched = await db.getOrg(org.id); - expect(fetched?.name).toBe("Renamed"); - // Slug should be unchanged - expect(fetched?.slug).toBe(org.slug); - // updated_at should be set - expect(fetched?.updatedAt).not.toBeNull(); - }); - - test("update org returns false for unknown id", async () => { - const result = await db.updateOrg("019d694f-79f6-7595-8faf-b70b01c11f98", { - name: "Whatever", - }); - expect(result).toBe(false); - }); - - test("update org with no fields is a no-op", async () => { - const org = await db.createOrg({ name: "No Fields" }); - const result = await db.updateOrg(org.id, {}); - expect(result).toBe(false); - - const fetched = await db.getOrg(org.id); - expect(fetched?.name).toBe("No Fields"); - }); - - test("list orgs by identity", async () => { - const identity = await db.createIdentity({ - email: "orglist@example.com", - name: "Org List User", - }); - - const org1 = await db.createOrg({ name: "Org 1" }); - const org2 = await db.createOrg({ name: "Org 2" }); - - await db.addMember(org1.id, identity.id, "owner"); - await db.addMember(org2.id, identity.id, "member"); - - const orgs = await db.listOrgsByIdentity(identity.id); - expect(orgs.length).toBe(2); - expect(orgs.map((o) => o.id).sort()).toEqual([org1.id, org2.id].sort()); - }); -}); - -// --------------------------------------------------------------------------- -// OrgMember tests -// --------------------------------------------------------------------------- - -describe("org-member", () => { - test("add and list members", async () => { - const org = await db.createOrg({ - name: "Member Test", - }); - const identity = await db.createIdentity({ - email: "member@example.com", - name: "Member", - }); - - const member = await db.addMember(org.id, identity.id, "admin"); - expect(member.orgId).toBe(org.id); - expect(member.identityId).toBe(identity.id); - expect(member.role).toBe("admin"); - - const members = await db.listMembers(org.id); - expect(members.length).toBe(1); - }); - - test("update role", async () => { - const org = await db.createOrg({ - name: "Role Update", - }); - const identity = await db.createIdentity({ - email: "roleupdate@example.com", - name: "Role Update", - }); - - await db.addMember(org.id, identity.id, "member"); - await db.updateRole(org.id, identity.id, "admin"); - - const member = await db.getMember(org.id, identity.id); - expect(member?.role).toBe("admin"); - }); - - test("cannot remove last owner", async () => { - const org = await db.createOrg({ name: "Last Owner" }); - const identity = await db.createIdentity({ - email: "lastowner@example.com", - name: "Last Owner", - }); - - await db.addMember(org.id, identity.id, "owner"); - - await expect(db.removeMember(org.id, identity.id)).rejects.toThrow( - AccountsError, - ); - }); - - test("cannot demote last owner", async () => { - const org = await db.createOrg({ - name: "Demote Owner", - }); - const identity = await db.createIdentity({ - email: "demoteowner@example.com", - name: "Demote Owner", - }); - - await db.addMember(org.id, identity.id, "owner"); - - await expect(db.updateRole(org.id, identity.id, "admin")).rejects.toThrow( - AccountsError, - ); - }); - - test("can remove owner if another owner exists", async () => { - const org = await db.createOrg({ - name: "Multi Owner", - }); - const owner1 = await db.createIdentity({ - email: "owner1@example.com", - name: "Owner 1", - }); - const owner2 = await db.createIdentity({ - email: "owner2@example.com", - name: "Owner 2", - }); - - await db.addMember(org.id, owner1.id, "owner"); - await db.addMember(org.id, owner2.id, "owner"); - - const removed = await db.removeMember(org.id, owner1.id); - expect(removed).toBe(true); - - const owners = await db.listOwners(org.id); - expect(owners.length).toBe(1); - expect(owners[0]?.identityId).toBe(owner2.id); - }); - - test("delete org cascades to org_member even when sole owner exists", async () => { - // Regression: previously the org_member_owner_check trigger refused - // the cascade, making it impossible to delete an org you owned. The - // invariant now lives in removeMember/updateRole so the cascade - // proceeds unimpeded. - const org = await db.createOrg({ name: "Sole Owner Cascade" }); - const identity = await db.createIdentity({ - email: "sole-owner-cascade@example.com", - name: "Sole Owner", - }); - await db.addMember(org.id, identity.id, "owner"); - - const deleted = await db.deleteOrg(org.id); - expect(deleted).toBe(true); - - const members = await db.listMembers(org.id); - expect(members.length).toBe(0); - }); -}); - -// --------------------------------------------------------------------------- -// Engine tests -// --------------------------------------------------------------------------- - -describe("engine", () => { - test("create engine with generated slug", async () => { - const org = await db.createOrg({ name: "Engine Org" }); - - const engine = await db.createEngine({ - orgId: org.id, - name: "My Engine", - }); - - expect(engine.id).toBeDefined(); - expect(engine.slug).toMatch(/^[a-z0-9]{12}$/); - expect(engine.name).toBe("My Engine"); - expect(engine.status).toBe("active"); - expect(engine.shardId).toBe(1); - }); - - test("get engine by slug", async () => { - const org = await db.createOrg({ - name: "Engine Slug", - }); - const engine = await db.createEngine({ - orgId: org.id, - name: "Slug Engine", - }); - - const fetched = await db.getEngineBySlug(engine.slug); - expect(fetched?.id).toBe(engine.id); - }); - - test("update engine status", async () => { - const org = await db.createOrg({ name: "Status" }); - const engine = await db.createEngine({ - orgId: org.id, - name: "Status Engine", - }); - - await db.updateEngine(engine.id, { status: "suspended" }); - - const fetched = await db.getEngine(engine.id); - expect(fetched?.status).toBe("suspended"); - }); - - test("update engine name", async () => { - const org = await db.createOrg({ name: "Rename" }); - const engine = await db.createEngine({ - orgId: org.id, - name: "Old Engine Name", - }); - - const updated = await db.updateEngine(engine.id, { - name: "New Engine Name", - }); - expect(updated).toBe(true); - - const fetched = await db.getEngine(engine.id); - expect(fetched?.name).toBe("New Engine Name"); - // Slug must remain stable across renames - expect(fetched?.slug).toBe(engine.slug); - expect(fetched?.updatedAt).not.toBeNull(); - }); - - test("update engine name conflicts with sibling in same org", async () => { - const org = await db.createOrg({ name: "Sibling Conflict" }); - await db.createEngine({ orgId: org.id, name: "Existing" }); - const target = await db.createEngine({ orgId: org.id, name: "Target" }); - - // Renaming "Target" to "Existing" should hit the (org_id, name) unique - // constraint and throw a Postgres error. - await expect( - db.updateEngine(target.id, { name: "Existing" }), - ).rejects.toThrow(); - }); - - test("list engines by org", async () => { - const org = await db.createOrg({ - name: "List Engines", - }); - await db.createEngine({ orgId: org.id, name: "Engine A" }); - await db.createEngine({ orgId: org.id, name: "Engine B" }); - - const engines = await db.listEnginesByOrg(org.id); - expect(engines.length).toBe(2); - }); - - test("createEngine stores custom language", async () => { - const org = await db.createOrg({ name: "Lang Org" }); - - const engine = await db.createEngine({ - orgId: org.id, - name: "German Engine", - language: "german", - }); - - expect(engine.language).toBe("german"); - - const fetched = await db.getEngine(engine.id); - expect(fetched?.language).toBe("german"); - }); - - test("createEngine defaults language to english", async () => { - const org = await db.createOrg({ - name: "Default Lang Org", - }); - - const engine = await db.createEngine({ - orgId: org.id, - name: "Default Engine", - }); - - expect(engine.language).toBe("english"); - }); -}); - -// --------------------------------------------------------------------------- -// Session tests -// --------------------------------------------------------------------------- - -describe("session", () => { - test("create and validate session", async () => { - const identity = await db.createIdentity({ - email: "session@example.com", - name: "Session User", - }); - - const { session, rawToken } = await db.createSession({ - identityId: identity.id, - }); - - expect(session.identityId).toBe(identity.id); - expect(rawToken).toBeDefined(); - - const result = await db.validateSession(rawToken); - expect(result?.session.id).toBe(session.id); - expect(result?.identity.id).toBe(identity.id); - }); - - test("invalid token returns null", async () => { - const result = await db.validateSession("invalid-token"); - expect(result).toBeNull(); - }); - - test("delete session", async () => { - const identity = await db.createIdentity({ - email: "deletesession@example.com", - name: "Delete Session", - }); - - const { session, rawToken } = await db.createSession({ - identityId: identity.id, - }); - - await db.deleteSession(session.id); - - const result = await db.validateSession(rawToken); - expect(result).toBeNull(); - }); - - test("delete all sessions for identity", async () => { - const identity = await db.createIdentity({ - email: "allsessions@example.com", - name: "All Sessions", - }); - - await db.createSession({ identityId: identity.id }); - await db.createSession({ identityId: identity.id }); - - const count = await db.deleteSessionsByIdentity(identity.id); - expect(count).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// Invitation tests -// --------------------------------------------------------------------------- - -describe("invitation", () => { - test("create and find invitation by token", async () => { - const org = await db.createOrg({ name: "Invite Org" }); - const inviter = await db.createIdentity({ - email: "inviter@example.com", - name: "Inviter", - }); - - const { invitation, rawToken } = await db.createInvitation({ - orgId: org.id, - email: "invitee@example.com", - role: "member", - invitedBy: inviter.id, - }); - - expect(invitation.orgId).toBe(org.id); - expect(invitation.email).toBe("invitee@example.com"); - expect(rawToken).toBeDefined(); - - const found = await db.getInvitationByToken(rawToken); - expect(found?.id).toBe(invitation.id); - }); - - test("accept invitation", async () => { - const org = await db.createOrg({ name: "Accept Org" }); - const inviter = await db.createIdentity({ - email: "acceptinviter@example.com", - name: "Inviter", - }); - - const { invitation } = await db.createInvitation({ - orgId: org.id, - email: "acceptee@example.com", - role: "admin", - invitedBy: inviter.id, - }); - - const accepted = await db.acceptInvitation(invitation.id); - expect(accepted?.acceptedAt).toBeDefined(); - }); - - test("list pending invitations", async () => { - const org = await db.createOrg({ - name: "Pending Org", - }); - const inviter = await db.createIdentity({ - email: "pendinginviter@example.com", - name: "Inviter", - }); - - await db.createInvitation({ - orgId: org.id, - email: "pending1@example.com", - role: "member", - invitedBy: inviter.id, - }); - await db.createInvitation({ - orgId: org.id, - email: "pending2@example.com", - role: "member", - invitedBy: inviter.id, - }); - - const pending = await db.listPendingInvitations(org.id); - expect(pending.length).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// OAuth tests -// --------------------------------------------------------------------------- - -describe("oauth", () => { - test("link and get oauth account", async () => { - const identity = await db.createIdentity({ - email: "oauth@example.com", - name: "OAuth User", - }); - - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-123", - email: "oauth@example.com", - }); - - expect(oauth.provider).toBe("github"); - expect(oauth.providerAccountId).toBe("gh-123"); - - const fetched = await db.getOAuthAccount("github", "gh-123"); - expect(fetched?.id).toBe(oauth.id); - }); - - test("list oauth accounts by identity", async () => { - const identity = await db.createIdentity({ - email: "multioauth@example.com", - name: "Multi OAuth", - }); - - await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-multi", - }); - await db.linkOAuthAccount({ - identityId: identity.id, - provider: "google", - providerAccountId: "google-multi", - }); - - const accounts = await db.getOAuthAccountsByIdentity(identity.id); - expect(accounts.length).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// Transaction test -// --------------------------------------------------------------------------- - -describe("transactions", () => { - test("withTransaction commits on success", async () => { - const result = await db.withTransaction(async (txDb) => { - const identity = await txDb.createIdentity({ - email: "txsuccess@example.com", - name: "TX Success", - }); - return identity; - }); - - const fetched = await db.getIdentity(result.id); - expect(fetched).toBeDefined(); - }); - - test("withTransaction rolls back on error", async () => { - const email = "txrollback@example.com"; - - try { - await db.withTransaction(async (txDb) => { - await txDb.createIdentity({ email, name: "TX Rollback" }); - throw new Error("Intentional error"); - }); - } catch { - // Expected - } - - const fetched = await db.getIdentityByEmail(email); - expect(fetched).toBeNull(); - }); -}); diff --git a/packages/accounts/db.ts b/packages/accounts/db.ts deleted file mode 100644 index aaf74ca..0000000 --- a/packages/accounts/db.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { SQL } from "bun"; -import { - type DeviceAuthOps, - deriveContext, - deviceAuthOps, - type EngineOps, - engineOps, - type IdentityOps, - type InvitationOps, - identityOps, - invitationOps, - type OAuthOps, - type OrgMemberOps, - type OrgOps, - oauthOps, - orgMemberOps, - orgOps, - type SessionOps, - sessionOps, - setLocalAccountsTimeouts, -} from "./ops"; -import type { AccountsContext } from "./types"; - -type AllOps = DeviceAuthOps & - IdentityOps & - OrgOps & - OrgMemberOps & - EngineOps & - InvitationOps & - OAuthOps & - SessionOps; - -export interface AccountsDB extends AllOps { - /** Execute operations within a transaction */ - withTransaction(fn: (db: AccountsDB) => Promise): Promise; -} - -function composeOps(ctx: AccountsContext): AllOps { - return { - ...deviceAuthOps(ctx), - ...identityOps(ctx), - ...orgOps(ctx), - ...orgMemberOps(ctx), - ...engineOps(ctx), - ...invitationOps(ctx), - ...oauthOps(ctx), - ...sessionOps(ctx), - }; -} - -export function createAccountsDB(sql: SQL, schema: string): AccountsDB { - const ctx: AccountsContext = { - sql, - schema, - inTransaction: false, - }; - - const ops = composeOps(ctx); - - const db: AccountsDB = { - ...ops, - - async withTransaction(fn: (db: AccountsDB) => Promise): Promise { - return sql.begin(async (tx) => { - await setLocalAccountsTimeouts(tx); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - - const txCtx = deriveContext(ctx, tx); - const txOps = composeOps(txCtx); - - const txDb: AccountsDB = { - ...txOps, - withTransaction: (nestedFn: (db: AccountsDB) => Promise) => - nestedFn(txDb), - }; - - return fn(txDb); - }); - }, - }; - - return db; -} diff --git a/packages/accounts/index.ts b/packages/accounts/index.ts deleted file mode 100644 index a1b80db..0000000 --- a/packages/accounts/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { type AccountsDB, createAccountsDB } from "./db"; -export * from "./types"; -export { generateToken, tokenHash } from "./util/hash"; -export { generateSlug } from "./util/slug"; diff --git a/packages/accounts/migrate/index.ts b/packages/accounts/migrate/index.ts deleted file mode 100644 index 29f70de..0000000 --- a/packages/accounts/migrate/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type { MigrateResult } from "./runner"; -export { dryRun, getMigrations, getVersion, migrate } from "./runner"; -export type { AccountsConfig, ResolvedConfig } from "./template"; -export { defaultConfig, resolveConfig, template } from "./template"; diff --git a/packages/accounts/migrate/migrate.integration.test.ts b/packages/accounts/migrate/migrate.integration.test.ts deleted file mode 100644 index 2417af0..0000000 --- a/packages/accounts/migrate/migrate.integration.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { - afterAll, - afterEach, - beforeAll, - describe, - expect, - test, -} from "bun:test"; -import { SQL } from "bun"; -import { dryRun, getMigrations, getVersion, migrate } from "./runner"; -import { - getDatabaseVersion, - schemaExists, - TestDatabase, - tableExists, -} from "./test-utils"; - -const adminUrl = "postgresql://postgres@localhost:5432/postgres"; - -describe("integration: migrate", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("creates schema and infrastructure tables on first run", async () => { - const schema = testSchema(); - const result = await migrate(sql, { schema }, "0.1.0"); - - expect(result.status).toBe("ok"); - expect(await schemaExists(sql, schema)).toBe(true); - expect(await tableExists(sql, schema, "version")).toBe(true); - expect(await tableExists(sql, schema, "migration")).toBe(true); - }); - - test("is idempotent", async () => { - const schema = testSchema(); - - const result1 = await migrate(sql, { schema }, "0.1.0"); - expect(result1.status).toBe("ok"); - - const result2 = await migrate(sql, { schema }, "0.1.0"); - expect(result2.status).toBe("ok"); - expect(result2.applied).toHaveLength(0); - }); - - test("tracks version correctly", async () => { - const schema = testSchema(); - - await migrate(sql, { schema }, "0.1.0"); - expect(await getDatabaseVersion(sql, schema)).toBe("0.1.0"); - - await migrate(sql, { schema }, "0.2.0"); - expect(await getDatabaseVersion(sql, schema)).toBe("0.2.0"); - }); - - test("rejects downgrade", async () => { - const schema = testSchema(); - - await migrate(sql, { schema }, "0.2.0"); - - try { - await migrate(sql, { schema }, "0.1.0"); - expect.unreachable("should have thrown"); - } catch (error) { - expect((error as Error).message).toContain("Server version (0.1.0)"); - expect((error as Error).message).toContain( - "older than database version (0.2.0)", - ); - } - }); - - test("version table has single-row constraint", async () => { - const schema = testSchema(); - await migrate(sql, { schema }, "0.1.0"); - - try { - await sql.unsafe( - `insert into ${schema}.version (version) values ('0.2.0')`, - ); - expect.unreachable("should have thrown"); - } catch (error) { - // Expected: unique constraint violation - expect(error).toBeTruthy(); - } - }); - - test("rejects migration by non-owner", async () => { - const schema = testSchema(); - - // First run creates the schema - await migrate(sql, { schema }, "0.1.0"); - - // Change owner to postgres (different from current user in some setups) - // This test may not trigger on all setups - the important thing is the code path exists - // The ownership check is in scaffold() - }); -}); - -describe("integration: dryRun", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("shows all pending for new schema", async () => { - const schema = testSchema(); - const result = await dryRun(sql, { schema }); - - // No migrations defined yet (scaffold handles infrastructure) - expect(result.pending.length).toBe(getMigrations().length); - expect(result.applied).toHaveLength(0); - }); - - test("shows none pending after migration", async () => { - const schema = testSchema(); - await migrate(sql, { schema }, "0.1.0"); - - const result = await dryRun(sql, { schema }); - - expect(result.pending).toHaveLength(0); - expect(result.applied.length).toBe(getMigrations().length); - }); -}); - -describe("integration: getVersion", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("returns current version", async () => { - const schema = testSchema(); - await migrate(sql, { schema }, "1.2.3"); - - const version = await getVersion(sql, { schema }); - expect(version).toBe("1.2.3"); - }); -}); - -describe("integration: TestDatabase", () => { - test("creates isolated schema", async () => { - const db = await TestDatabase.create(adminUrl, "0.1.0"); - - try { - expect(db.schema).toMatch(/^accounts_test_/); - expect(await schemaExists(db.sql, db.schema)).toBe(true); - expect(await tableExists(db.sql, db.schema, "migration")).toBe(true); - expect(await tableExists(db.sql, db.schema, "version")).toBe(true); - } finally { - await db.dispose(); - } - }); - - test("dispose drops schema", async () => { - const db = await TestDatabase.create(adminUrl, "0.1.0"); - const schema = db.schema; - - const sql = new SQL(adminUrl); - try { - expect(await schemaExists(sql, schema)).toBe(true); - await db.dispose(); - expect(await schemaExists(sql, schema)).toBe(false); - } finally { - await sql.close(); - } - }); -}); - -describe("integration: advisory locks", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("concurrent migrations on same schema - handles gracefully", async () => { - const schema = testSchema(); - - const results = await Promise.all([ - migrate(sql, { schema }, "0.1.0"), - migrate(sql, { schema }, "0.1.0"), - migrate(sql, { schema }, "0.1.0"), - ]); - - // All should complete (ok or skipped) - for (const result of results) { - expect(["ok", "skipped"]).toContain(result.status); - } - - // Schema should exist and be properly set up - expect(await schemaExists(sql, schema)).toBe(true); - expect(await tableExists(sql, schema, "version")).toBe(true); - }); -}); diff --git a/packages/accounts/migrate/migrations/001_updated_at.sql b/packages/accounts/migrate/migrations/001_updated_at.sql deleted file mode 100644 index f2c6856..0000000 --- a/packages/accounts/migrate/migrations/001_updated_at.sql +++ /dev/null @@ -1,13 +0,0 @@ --- extensions required by the accounts schema -create extension if not exists citext; - --- generic trigger function to update updated_at timestamp -create function {{schema}}.update_updated_at() -returns trigger -as $func$ -begin - new.updated_at = pg_catalog.now(); - return new; -end; -$func$ language plpgsql volatile security definer -set search_path to {{schema}}, pg_temp; diff --git a/packages/accounts/migrate/migrations/002_core_tables.sql b/packages/accounts/migrate/migrations/002_core_tables.sql deleted file mode 100644 index 92a40b1..0000000 --- a/packages/accounts/migrate/migrations/002_core_tables.sql +++ /dev/null @@ -1,58 +0,0 @@ --- ===== Shard (minimal, for future scaling) ===== -create table {{schema}}.shard -( id int primary key -); - --- seed default shard -insert into {{schema}}.shard (id) values (1); - --- ===== Org (billing/ownership entity) ===== -create table {{schema}}.org -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, slug text unique not null check (slug ~ '^[a-z0-9]{12}$') -, name text not null -, created_at timestamptz not null default now() -, updated_at timestamptz -); - -create trigger org_updated_at - before update on {{schema}}.org - for each row - execute function {{schema}}.update_updated_at(); - --- ===== Identity (human who can log in) ===== -create table {{schema}}.identity -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, email citext unique not null -, name text not null -, created_at timestamptz not null default now() -, updated_at timestamptz -); - -create trigger identity_updated_at - before update on {{schema}}.identity - for each row - execute function {{schema}}.update_updated_at(); - --- ===== Engine (memory engine instance) ===== -create table {{schema}}.engine -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, org_id uuid not null references {{schema}}.org on delete cascade -, slug text unique not null check (slug ~ '^[a-z0-9]{12}$') -, name text not null -, shard_id int not null references {{schema}}.shard -, status text not null default 'active' check (status in ('active', 'suspended', 'deleted')) -, language text not null default 'english' check (language ~ '^[a-z_]+$') -, created_at timestamptz not null default now() -, updated_at timestamptz -, unique (org_id, name) -); - -create index idx_engine_org on {{schema}}.engine (org_id); -create index idx_engine_shard on {{schema}}.engine (shard_id); -create index idx_engine_status on {{schema}}.engine (status) where status <> 'active'; - -create trigger engine_updated_at - before update on {{schema}}.engine - for each row - execute function {{schema}}.update_updated_at(); diff --git a/packages/accounts/migrate/migrations/003_membership.sql b/packages/accounts/migrate/migrations/003_membership.sql deleted file mode 100644 index 9feba1a..0000000 --- a/packages/accounts/migrate/migrations/003_membership.sql +++ /dev/null @@ -1,10 +0,0 @@ --- ===== Org Member (identity membership in org) ===== -create table {{schema}}.org_member -( org_id uuid not null references {{schema}}.org on delete cascade -, identity_id uuid not null references {{schema}}.identity on delete cascade -, role text not null check (role in ('owner', 'admin', 'member')) -, created_at timestamptz not null default now() -, primary key (org_id, identity_id) -); - -create index idx_org_member_identity on {{schema}}.org_member (identity_id); diff --git a/packages/accounts/migrate/migrations/004_invitations.sql b/packages/accounts/migrate/migrations/004_invitations.sql deleted file mode 100644 index 130682c..0000000 --- a/packages/accounts/migrate/migrations/004_invitations.sql +++ /dev/null @@ -1,17 +0,0 @@ --- ===== Invitation (pending org invitations) ===== -create table {{schema}}.invitation -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, org_id uuid not null references {{schema}}.org on delete cascade -, email citext not null -, role text not null check (role in ('owner', 'admin', 'member')) -, token text unique not null -, invited_by uuid not null references {{schema}}.identity -, expires_at timestamptz not null -, accepted_at timestamptz -, created_at timestamptz not null default now() -, unique (org_id, email) -); - -create index idx_invitation_token on {{schema}}.invitation (token) where accepted_at is null; -create index idx_invitation_org on {{schema}}.invitation (org_id) where accepted_at is null; -create index idx_invitation_email on {{schema}}.invitation (email) where accepted_at is null; diff --git a/packages/accounts/migrate/migrations/005_auth.sql b/packages/accounts/migrate/migrations/005_auth.sql deleted file mode 100644 index 27545c0..0000000 --- a/packages/accounts/migrate/migrations/005_auth.sql +++ /dev/null @@ -1,34 +0,0 @@ --- ===== OAuth Account (provider links) ===== -create table {{schema}}.oauth_account -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, identity_id uuid not null references {{schema}}.identity on delete cascade -, provider text not null check (provider in ('google', 'github')) -, provider_account_id text not null -, email citext -, access_token text -, refresh_token text -, token_expires_at timestamptz -, created_at timestamptz not null default now() -, updated_at timestamptz -, unique (provider, provider_account_id) -); - -create index idx_oauth_account_identity on {{schema}}.oauth_account (identity_id); - -create trigger oauth_account_updated_at - before update on {{schema}}.oauth_account - for each row - execute function {{schema}}.update_updated_at(); - --- ===== Session (for OAuth flow) ===== -create table {{schema}}.session -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, identity_id uuid not null references {{schema}}.identity on delete cascade -, token text unique not null -, expires_at timestamptz not null -, created_at timestamptz not null default now() -); - -create index idx_session_identity on {{schema}}.session (identity_id); -create index idx_session_token on {{schema}}.session (token); -create index idx_session_expires on {{schema}}.session (expires_at); diff --git a/packages/accounts/migrate/migrations/006_ops_support.sql b/packages/accounts/migrate/migrations/006_ops_support.sql deleted file mode 100644 index 90dbda2..0000000 --- a/packages/accounts/migrate/migrations/006_ops_support.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Encryption keys for envelope encryption -create table {{schema}}.encryption_key -( id int primary key generated always as identity -, key_ciphertext bytea not null -, active boolean not null default false -, created_at timestamptz not null default now() -); - --- Only one active key at a time -create unique index idx_encryption_key_active - on {{schema}}.encryption_key (active) where active = true; - --- Track which key encrypted OAuth tokens -alter table {{schema}}.oauth_account - add column encryption_key_id int references {{schema}}.encryption_key(id); - --- Trigger to prevent removing the last owner from an org -create function {{schema}}.check_org_has_owner() -returns trigger as $func$ -begin - if (TG_OP = 'DELETE' and OLD.role = 'owner') - or (TG_OP = 'UPDATE' and OLD.role = 'owner' and NEW.role <> 'owner') - then - if not exists ( - select 1 from {{schema}}.org_member - where org_id = OLD.org_id - and role = 'owner' - and identity_id <> OLD.identity_id - ) then - raise exception 'org_must_have_owner' - using errcode = 'P0001', - hint = 'Cannot remove the last owner from an organization'; - end if; - end if; - - if TG_OP = 'DELETE' then - return OLD; - end if; - return NEW; -end; -$func$ language plpgsql; - -create trigger org_member_owner_check - before delete or update on {{schema}}.org_member - for each row - execute function {{schema}}.check_org_has_owner(); diff --git a/packages/accounts/migrate/migrations/007_device_authorization.sql b/packages/accounts/migrate/migrations/007_device_authorization.sql deleted file mode 100644 index 476284d..0000000 --- a/packages/accounts/migrate/migrations/007_device_authorization.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Device authorization for OAuth device flow (RFC 8628) --- Stores state during the ~15 minute device flow window - -create table {{schema}}.device_authorization ( - device_code text primary key, -- URL-safe base64, 32 bytes - user_code text not null unique, -- XXXX-XXXX format - provider text not null, -- 'google' | 'github' - oauth_state text not null unique, -- CSRF protection - expires_at timestamptz not null, -- 15 minute TTL - last_poll timestamptz, -- rate limiting - identity_id uuid references {{schema}}.identity(id) on delete cascade, -- set when authorized - denied boolean not null default false, -- user denied access - created_at timestamptz not null default now() -); diff --git a/packages/accounts/migrate/migrations/008_drop_org_owner_trigger.sql b/packages/accounts/migrate/migrations/008_drop_org_owner_trigger.sql deleted file mode 100644 index 260efbf..0000000 --- a/packages/accounts/migrate/migrations/008_drop_org_owner_trigger.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Remove the "must keep at least one owner" trigger on org_member. --- --- The trigger fired before any DELETE or UPDATE on org_member, including --- the rows cascaded by `delete from org` (since org_member.org_id has --- `references org on delete cascade`). When an org was being deleted in --- its entirety, the cascade removed the org's owner row too, and the --- trigger refused — making it impossible to delete an org that you owned. --- --- The invariant is now enforced at the application layer in --- packages/accounts/ops/org-member.ts (removeMember + updateRole), where --- it can distinguish member-management flows from a cascading org delete. -drop trigger if exists org_member_owner_check on {{schema}}.org_member; -drop function if exists {{schema}}.check_org_has_owner(); diff --git a/packages/accounts/migrate/migrations/009_session_lookup.sql b/packages/accounts/migrate/migrations/009_session_lookup.sql deleted file mode 100644 index c1097dd..0000000 --- a/packages/accounts/migrate/migrations/009_session_lookup.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Session and invitation tokens are stored as their sha256 digest in --- token_hash, used as a unique-indexed lookup key. Raw tokens are 256-bit --- CSPRNG output, so sha256 alone provides preimage resistance equivalent --- to argon2 against an offline DB dump — without paying ~60ms per verify --- and without the O(n) scan-and-verify pattern the previous schema forced. - --- Drop existing rows: we don't store raw tokens, so we cannot derive --- token_hash for them. All current CLI sessions become invalid (next --- command yields a 401, "Invalid or expired session"; user runs `me login`). --- All pending invitations must be re-issued. -truncate {{schema}}.session; -truncate {{schema}}.invitation; - --- `drop column` cascades to dependent indexes and constraints. This removes --- session_token_key (unique constraint) and idx_session_token, plus --- invitation_token_key (unique constraint) and idx_invitation_token. -alter table {{schema}}.session drop column token; -alter table {{schema}}.invitation drop column token; - -alter table {{schema}}.session - add column token_hash bytea not null; -alter table {{schema}}.invitation - add column token_hash bytea not null; - -create unique index session_token_hash_uniq - on {{schema}}.session (token_hash); - -create unique index invitation_token_hash_uniq - on {{schema}}.invitation (token_hash) - where accepted_at is null; diff --git a/packages/accounts/migrate/migrations/sql.d.ts b/packages/accounts/migrate/migrations/sql.d.ts deleted file mode 100644 index 89b092e..0000000 --- a/packages/accounts/migrate/migrations/sql.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -declare module "*.sql" { - const content: string; - export default content; -} diff --git a/packages/accounts/migrate/runner.test.ts b/packages/accounts/migrate/runner.test.ts deleted file mode 100644 index 6ffc14f..0000000 --- a/packages/accounts/migrate/runner.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { getMigrations } from "./runner"; - -describe("getMigrations", () => { - test("returns an array", () => { - expect(Array.isArray(getMigrations())).toBe(true); - }); - - test("migrations are sorted by name", () => { - const names = getMigrations().map((m) => m.name); - const sorted = [...names].sort(); - expect(names).toEqual(sorted); - }); - - test("migration names match NNN_name pattern", () => { - for (const { name } of getMigrations()) { - expect(name).toMatch(/^\d{3}_\w+$/); - } - }); - - // Note: scaffold() handles infrastructure (schema, version, migration tables) - // Domain migrations will be added to the migrations array as needed -}); diff --git a/packages/accounts/migrate/runner.ts b/packages/accounts/migrate/runner.ts deleted file mode 100644 index 8cf8ca8..0000000 --- a/packages/accounts/migrate/runner.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { type SQL, semver } from "bun"; -import { setLocalAccountsTimeouts } from "../ops/_tx"; -import migration001 from "./migrations/001_updated_at.sql" with { - type: "text", -}; -import migration002 from "./migrations/002_core_tables.sql" with { - type: "text", -}; -import migration003 from "./migrations/003_membership.sql" with { - type: "text", -}; -import migration004 from "./migrations/004_invitations.sql" with { - type: "text", -}; -import migration005 from "./migrations/005_auth.sql" with { type: "text" }; -import migration006 from "./migrations/006_ops_support.sql" with { - type: "text", -}; -import migration007 from "./migrations/007_device_authorization.sql" with { - type: "text", -}; -import migration008 from "./migrations/008_drop_org_owner_trigger.sql" with { - type: "text", -}; -import migration009 from "./migrations/009_session_lookup.sql" with { - type: "text", -}; -import { type AccountsConfig, resolveConfig, template } from "./template"; - -interface Migration { - name: string; - sql: string; -} - -const migrations: Migration[] = [ - { name: "001_updated_at", sql: migration001 }, - { name: "002_core_tables", sql: migration002 }, - { name: "003_membership", sql: migration003 }, - { name: "004_invitations", sql: migration004 }, - { name: "005_auth", sql: migration005 }, - { name: "006_ops_support", sql: migration006 }, - { name: "007_device_authorization", sql: migration007 }, - { name: "008_drop_org_owner_trigger", sql: migration008 }, - { name: "009_session_lookup", sql: migration009 }, -]; - -export interface MigrateResult { - schema: string; - status: "ok" | "skipped" | "error"; - applied: string[]; - error?: Error; -} - -const MAX_LOCK_RETRIES = 5; -const BASE_DELAY_MS = 100; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** - * Scaffold creates the migration infrastructure: schema, version table, migration table. - * This runs before migrations and is idempotent - safe to call multiple times. - * Also validates ownership to prevent migrating a schema you don't own. - */ -async function scaffold(tx: SQL, schema: string): Promise { - await tx.unsafe(` - do $block$ - declare - _owner oid; - _user oid; - begin - select pg_catalog.to_regrole(current_user)::oid - into strict _user - ; - - select n.nspowner into _owner - from pg_catalog.pg_namespace n - where n.nspname = '${schema}' - ; - - if _owner is null then - -- schema doesn't exist, create infrastructure - create schema ${schema}; - - -- version table (single row, tracks overall schema version) - create table ${schema}.version - ( version text not null check (version ~ '^\\d+\\.\\d+\\.\\d+$') - , at timestamptz not null default now() - ); - create unique index on ${schema}.version ((true)); - insert into ${schema}.version (version) values ('0.0.0'); - - -- migration table - create table ${schema}.migration - ( name text not null primary key - , applied_at_version text not null - , applied_at timestamptz not null default pg_catalog.clock_timestamp() - ); - - elsif _owner is distinct from _user then - raise exception 'only the owner of the ${schema} schema can run database migrations'; - end if - ; - end - $block$ - `); -} - -export async function migrate( - sql: SQL, - config?: AccountsConfig, - serverVersion = "0.0.0", -): Promise { - const resolved = resolveConfig(config); - const { schema } = resolved; - - return await sql.begin(async (tx) => { - await setLocalAccountsTimeouts(tx); - - // Acquire advisory lock with retry - const [{ lock_id }] = - await tx`select hashtext(${schema})::bigint as lock_id`; - - let acquired = false; - for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { - const [result] = - await tx`select pg_try_advisory_xact_lock(${lock_id}) as acquired`; - if (result.acquired) { - acquired = true; - break; - } - if (attempt < MAX_LOCK_RETRIES - 1) { - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } - - if (!acquired) { - return { schema, status: "skipped" as const, applied: [] }; - } - - // Scaffold creates schema + version + migration tables (idempotent) - await scaffold(tx, schema); - - // Check version - reject downgrades - const [{ version: dbVersion }] = await tx.unsafe( - `select version from ${schema}.version`, - ); - - const cmp = semver.order(serverVersion, dbVersion); - if (cmp < 0) { - throw new Error( - `Server version (${serverVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the server.", - ); - } - - // Run migrations - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - const applied: string[] = []; - - for (const migration of sorted) { - const [existing] = await tx.unsafe( - `select 1 from ${schema}.migration where name = $1`, - [migration.name], - ); - - if (existing) { - continue; - } - - const renderedSql = template(migration.sql, resolved); - await tx.unsafe(renderedSql); - await tx.unsafe( - `insert into ${schema}.migration (name, applied_at_version) values ($1, $2)`, - [migration.name, serverVersion], - ); - applied.push(migration.name); - } - - // Update version if app version is newer - if (cmp > 0) { - await tx.unsafe(`update ${schema}.version set version = $1, at = now()`, [ - serverVersion, - ]); - } - - return { schema, status: "ok" as const, applied }; - }); -} - -export async function dryRun( - sql: SQL, - config?: AccountsConfig, -): Promise<{ pending: string[]; applied: string[] }> { - const { schema } = resolveConfig(config); - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - - // Check if migration table exists - const [{ exists }] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = 'migration' - ) as exists - `; - - if (!exists) { - return { - pending: sorted.map((m) => m.name), - applied: [], - }; - } - - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - const appliedSet = new Set(rows.map((r: { name: string }) => r.name)); - const applied = sorted - .filter((m) => appliedSet.has(m.name)) - .map((m) => m.name); - const pending = sorted - .filter((m) => !appliedSet.has(m.name)) - .map((m) => m.name); - - return { pending, applied }; -} - -export async function getVersion( - sql: SQL, - config?: AccountsConfig, -): Promise { - const { schema } = resolveConfig(config); - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row.version; -} - -export function getMigrations(): ReadonlyArray<{ name: string }> { - return [...migrations] - .sort((a, b) => a.name.localeCompare(b.name)) - .map(({ name }) => ({ name })); -} diff --git a/packages/accounts/migrate/template.test.ts b/packages/accounts/migrate/template.test.ts deleted file mode 100644 index 89fe385..0000000 --- a/packages/accounts/migrate/template.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { defaultConfig, resolveConfig, template } from "./template"; - -describe("template function", () => { - test("replaces single variable", () => { - const sql = "CREATE TABLE {{schema}}.foo (id uuid)"; - const result = template(sql, { schema: "accounts" }); - expect(result).toBe("CREATE TABLE accounts.foo (id uuid)"); - }); - - test("replaces same variable multiple times", () => { - const sql = "{{schema}}.a and {{schema}}.b"; - const result = template(sql, { schema: "test" }); - expect(result).toBe("test.a and test.b"); - }); - - test("throws on missing variable", () => { - const sql = "CREATE TABLE {{missing}}.foo"; - expect(() => template(sql, {})).toThrow( - "Missing template variable: missing", - ); - }); - - test("handles no variables", () => { - const sql = "CREATE TABLE foo (id uuid)"; - const result = template(sql, {}); - expect(result).toBe("CREATE TABLE foo (id uuid)"); - }); - - test("handles numeric values", () => { - const sql = "LIMIT {{limit}}"; - const result = template(sql, { limit: 100 }); - expect(result).toBe("LIMIT 100"); - }); -}); - -describe("config", () => { - test("defaultConfig has schema = accounts", () => { - expect(defaultConfig.schema).toBe("accounts"); - }); - - test("resolveConfig uses default schema", () => { - const resolved = resolveConfig(); - expect(resolved.schema).toBe("accounts"); - }); - - test("resolveConfig allows schema override", () => { - const resolved = resolveConfig({ schema: "accounts_test" }); - expect(resolved.schema).toBe("accounts_test"); - }); -}); diff --git a/packages/accounts/migrate/template.ts b/packages/accounts/migrate/template.ts deleted file mode 100644 index d5a6194..0000000 --- a/packages/accounts/migrate/template.ts +++ /dev/null @@ -1,22 +0,0 @@ -export function template(sql: string, vars: Record): string { - return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { - if (!(key in vars)) { - throw new Error(`Missing template variable: ${key}`); - } - return String(vars[key]); - }); -} - -export interface AccountsConfig { - schema?: string; -} - -export type ResolvedConfig = Required; - -export const defaultConfig: ResolvedConfig = { - schema: "accounts", -}; - -export function resolveConfig(config?: AccountsConfig): ResolvedConfig { - return { ...defaultConfig, ...config }; -} diff --git a/packages/accounts/migrate/test-utils.ts b/packages/accounts/migrate/test-utils.ts deleted file mode 100644 index 36271a5..0000000 --- a/packages/accounts/migrate/test-utils.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { SQL } from "bun"; -import { migrate } from "./runner"; -import type { AccountsConfig } from "./template"; - -function assertSafeIdentifier(name: string): void { - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { - throw new Error(`Unsafe database identifier: ${name}`); - } -} - -export class TestDatabase { - schema: string; - sql: SQL; - - private constructor(schema: string, sql: SQL) { - this.schema = schema; - this.sql = sql; - } - - static async create( - adminUrl = "postgresql://postgres@localhost:5432/postgres", - serverVersion = "0.1.0", - ): Promise { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - assertSafeIdentifier(schema); - - const sql = new SQL(adminUrl); - const config: AccountsConfig = { schema }; - - await migrate(sql, config, serverVersion); - - return new TestDatabase(schema, sql); - } - - async dispose(): Promise { - assertSafeIdentifier(this.schema); - await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); - await this.sql.close(); - } -} - -export async function getAppliedMigrations( - sql: SQL, - schema: string, -): Promise { - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - return rows.map((r: { name: string }) => r.name); -} - -export async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = ${table} - ) as exists - `; - return row.exists; -} - -export async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.schemata - where schema_name = ${name} - ) as exists - `; - return row.exists; -} - -export async function getDatabaseVersion( - sql: SQL, - schema: string, -): Promise { - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row.version; -} diff --git a/packages/accounts/ops/_tx.ts b/packages/accounts/ops/_tx.ts deleted file mode 100644 index 1b73e76..0000000 --- a/packages/accounts/ops/_tx.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Transaction helper for accounts operations - * - * Simpler than engine's _tx.ts - no RLS roles needed since accounts - * uses application-level authorization. - */ - -import { span } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import type { AccountsContext } from "../types"; - -const ACCOUNTS_STATEMENT_TIMEOUT = - process.env.ACCOUNTS_STATEMENT_TIMEOUT ?? "25s"; -const ACCOUNTS_LOCK_TIMEOUT = process.env.ACCOUNTS_LOCK_TIMEOUT ?? "5s"; -const ACCOUNTS_TRANSACTION_TIMEOUT = - process.env.ACCOUNTS_TRANSACTION_TIMEOUT ?? "30s"; -const ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT = - process.env.ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? "30s"; - -/** - * Bound accounts transactions with transaction-local GUCs so pooled - * connections do not retain settings. - */ -export async function setLocalAccountsTimeouts(sql: SQL): Promise { - await sql.unsafe("SELECT set_config('statement_timeout', $1, true)", [ - ACCOUNTS_STATEMENT_TIMEOUT, - ]); - await sql.unsafe("SELECT set_config('lock_timeout', $1, true)", [ - ACCOUNTS_LOCK_TIMEOUT, - ]); - await sql.unsafe("SELECT set_config('transaction_timeout', $1, true)", [ - ACCOUNTS_TRANSACTION_TIMEOUT, - ]); - await sql.unsafe( - "SELECT set_config('idle_in_transaction_session_timeout', $1, true)", - [ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT], - ); -} - -/** - * Execute a function within a transaction context. - * - * If already inside a transaction (ctx.inTransaction is true), runs directly. - * Otherwise opens a new transaction with search_path set. - */ -export async function withTx( - ctx: AccountsContext, - operation: string, - fn: (sql: SQL) => Promise, -): Promise { - if (ctx.inTransaction) { - return fn(ctx.sql); - } - - return span(`accounts.${operation}`, { - attributes: { - "db.schema": ctx.schema, - "db.operation": operation, - }, - callback: () => - ctx.sql.begin(async (tx) => { - await setLocalAccountsTimeouts(tx); - await tx.unsafe(`SET LOCAL search_path TO ${ctx.schema}, public`); - return fn(tx); - }), - }); -} - -/** - * Create a derived context for use inside withTransaction() - */ -export function deriveContext(ctx: AccountsContext, tx: SQL): AccountsContext { - return { - ...ctx, - sql: tx, - inTransaction: true, - }; -} diff --git a/packages/accounts/ops/device-auth.ts b/packages/accounts/ops/device-auth.ts deleted file mode 100644 index 881f666..0000000 --- a/packages/accounts/ops/device-auth.ts +++ /dev/null @@ -1,207 +0,0 @@ -import type { - AccountsContext, - CreateDeviceAuthParams, - DeviceAuthorization, - DeviceProvider, -} from "../types"; -import { withTx } from "./_tx"; - -interface DeviceAuthRow { - device_code: string; - user_code: string; - provider: string; - oauth_state: string; - expires_at: Date; - last_poll: Date | null; - identity_id: string | null; - denied: boolean; - created_at: Date; -} - -function rowToDeviceAuth(row: DeviceAuthRow): DeviceAuthorization { - return { - deviceCode: row.device_code, - userCode: row.user_code, - provider: row.provider as DeviceProvider, - oauthState: row.oauth_state, - expiresAt: row.expires_at, - lastPoll: row.last_poll, - identityId: row.identity_id, - denied: row.denied, - createdAt: row.created_at, - }; -} - -export function deviceAuthOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - /** - * Create a new device authorization. - */ - async create(params: CreateDeviceAuthParams): Promise { - const { deviceCode, userCode, provider, oauthState, expiresAt } = params; - - return withTx(ctx, "createDeviceAuth", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.device_authorization - (device_code, user_code, provider, oauth_state, expires_at) - values - (${deviceCode}, ${userCode}, ${provider}, ${oauthState}, ${expiresAt}) - returning * - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create device authorization"); - } - return rowToDeviceAuth(row); - }); - }, - - /** - * Get device authorization by device code (for CLI polling). - * Returns null if not found or expired. - */ - async getByDeviceCode( - deviceCode: string, - ): Promise { - return withTx(ctx, "getDeviceByCode", async (sql) => { - const rows = await sql` - select * from ${sql.unsafe(schema)}.device_authorization - where device_code = ${deviceCode} - and expires_at > now() - `; - const row = rows[0]; - return row ? rowToDeviceAuth(row) : null; - }); - }, - - /** - * Get device authorization by user code (for browser code entry). - * Normalizes input: uppercase, removes hyphens, reconstructs format. - * Returns null if not found or expired. - */ - async getByUserCode(userCode: string): Promise { - // Normalize: uppercase, remove hyphen, reconstruct XXXX-XXXX - const normalized = userCode.toUpperCase().replace(/-/g, ""); - const formatted = `${normalized.slice(0, 4)}-${normalized.slice(4)}`; - - return withTx(ctx, "getDeviceByUserCode", async (sql) => { - const rows = await sql` - select * from ${sql.unsafe(schema)}.device_authorization - where user_code = ${formatted} - and expires_at > now() - `; - const row = rows[0]; - return row ? rowToDeviceAuth(row) : null; - }); - }, - - /** - * Get device authorization by OAuth state (for callback). - * Returns null if not found or expired. - */ - async getByOAuthState( - oauthState: string, - ): Promise { - return withTx(ctx, "getDeviceByOAuthState", async (sql) => { - const rows = await sql` - select * from ${sql.unsafe(schema)}.device_authorization - where oauth_state = ${oauthState} - and expires_at > now() - `; - const row = rows[0]; - return row ? rowToDeviceAuth(row) : null; - }); - }, - - /** - * Update last poll time for rate limiting. - * Returns the time since last poll in milliseconds, or null if first poll. - */ - async updateLastPoll(deviceCode: string): Promise { - return withTx(ctx, "updateDeviceLastPoll", async (sql) => { - const rows = await sql<{ last_poll: Date | null }[]>` - update ${sql.unsafe(schema)}.device_authorization - set last_poll = now() - where device_code = ${deviceCode} - and expires_at > now() - returning ( - select last_poll from ${sql.unsafe(schema)}.device_authorization - where device_code = ${deviceCode} - ) as last_poll - `; - const row = rows[0]; - if (!row?.last_poll) { - return null; // First poll or not found - } - return Date.now() - row.last_poll.getTime(); - }); - }, - - /** - * Mark device as authorized with an identity. - * Returns true if updated, false if not found/expired. - */ - async authorize(deviceCode: string, identityId: string): Promise { - return withTx(ctx, "authorizeDevice", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.device_authorization - set identity_id = ${identityId} - where device_code = ${deviceCode} - and expires_at > now() - and identity_id is null - and denied = false - `; - return result.count > 0; - }); - }, - - /** - * Mark device as denied. - * Returns true if updated, false if not found/expired. - */ - async deny(deviceCode: string): Promise { - return withTx(ctx, "denyDevice", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.device_authorization - set denied = true - where device_code = ${deviceCode} - and expires_at > now() - and identity_id is null - `; - return result.count > 0; - }); - }, - - /** - * Delete a device authorization (cleanup after completion). - * Returns true if deleted. - */ - async delete(deviceCode: string): Promise { - return withTx(ctx, "deleteDevice", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.device_authorization - where device_code = ${deviceCode} - `; - return result.count > 0; - }); - }, - - /** - * Delete all expired device authorizations. - * Called by cron job. Returns count deleted. - */ - async deleteExpired(): Promise { - return withTx(ctx, "deleteExpiredDevices", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.device_authorization - where expires_at <= now() - `; - return result.count; - }); - }, - }; -} - -export type DeviceAuthOps = ReturnType; diff --git a/packages/accounts/ops/engine.ts b/packages/accounts/ops/engine.ts deleted file mode 100644 index 64437aa..0000000 --- a/packages/accounts/ops/engine.ts +++ /dev/null @@ -1,143 +0,0 @@ -import type { - AccountsContext, - CreateEngineParams, - Engine, - EngineStatus, -} from "../types"; -import { generateSlug } from "../util/slug"; -import { withTx } from "./_tx"; - -interface EngineRow { - id: string; - org_id: string; - slug: string; - name: string; - shard_id: number; - status: EngineStatus; - language: string; - created_at: Date; - updated_at: Date | null; -} - -function rowToEngine(row: EngineRow): Engine { - return { - id: row.id, - orgId: row.org_id, - slug: row.slug, - name: row.name, - shardId: row.shard_id, - status: row.status, - language: row.language, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function engineOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createEngine(params: CreateEngineParams): Promise { - const { id, orgId, name, shardId = 1, language = "english" } = params; - const slug = generateSlug(); - - return withTx(ctx, "createEngine", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.engine (id, org_id, slug, name, shard_id, language) - values (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${orgId}, ${slug}, ${name}, ${shardId}, ${language}) - returning id, org_id, slug, name, shard_id, status, language, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create engine"); - } - return rowToEngine(row); - }); - }, - - async getEngine(id: string): Promise { - return withTx(ctx, "getEngine", async (sql) => { - const [row] = await sql` - select id, org_id, slug, name, shard_id, status, language, created_at, updated_at - from ${sql.unsafe(schema)}.engine - where id = ${id} - `; - return row ? rowToEngine(row) : null; - }); - }, - - async getEngineBySlug(slug: string): Promise { - return withTx(ctx, "getEngineBySlug", async (sql) => { - const [row] = await sql` - select id, org_id, slug, name, shard_id, status, language, created_at, updated_at - from ${sql.unsafe(schema)}.engine - where slug = ${slug} - `; - return row ? rowToEngine(row) : null; - }); - }, - - async updateEngine( - id: string, - params: { name?: string; status?: EngineStatus }, - ): Promise { - const { name, status } = params; - if (name === undefined && status === undefined) { - return false; - } - - return withTx(ctx, "updateEngine", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.engine - set - ${name !== undefined ? sql`name = ${name},` : sql``} - ${status !== undefined ? sql`status = ${status},` : sql``} - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async listEnginesByOrg(orgId: string): Promise { - return withTx(ctx, "listEnginesByOrg", async (sql) => { - const rows = await sql` - select id, org_id, slug, name, shard_id, status, language, created_at, updated_at - from ${sql.unsafe(schema)}.engine - where org_id = ${orgId} - order by created_at - `; - return rows.map(rowToEngine); - }); - }, - - /** - * Hard-delete an engine row. Returns true if a row was deleted, false - * if no row matched. Caller is responsible for first dropping the - * engine schema; this only removes the accounts-side metadata. - */ - async deleteEngine(id: string): Promise { - return withTx(ctx, "deleteEngine", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.engine - where id = ${id} - `; - return result.count > 0; - }); - }, - - /** List all active engines across all orgs (for embedding worker discovery) */ - async listActiveEngines(): Promise<{ slug: string; shardId: number }[]> { - return withTx(ctx, "listActiveEngines", async (sql) => { - const rows = await sql<{ slug: string; shard_id: number }[]>` - select slug, shard_id - from ${sql.unsafe(schema)}.engine - where status = 'active' - `; - return rows.map((r) => ({ slug: r.slug, shardId: r.shard_id })); - }); - }, - }; -} - -export type EngineOps = ReturnType; diff --git a/packages/accounts/ops/identity.ts b/packages/accounts/ops/identity.ts deleted file mode 100644 index 04057ed..0000000 --- a/packages/accounts/ops/identity.ts +++ /dev/null @@ -1,99 +0,0 @@ -import type { AccountsContext, CreateIdentityParams, Identity } from "../types"; -import { withTx } from "./_tx"; - -interface IdentityRow { - id: string; - email: string; - name: string; - created_at: Date; - updated_at: Date | null; -} - -function rowToIdentity(row: IdentityRow): Identity { - return { - id: row.id, - email: row.email, - name: row.name, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function identityOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createIdentity(params: CreateIdentityParams): Promise { - const { id, email, name } = params; - - return withTx(ctx, "createIdentity", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.identity (id, email, name) - values (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${email}, ${name}) - returning id, email, name, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create identity"); - } - return rowToIdentity(row); - }); - }, - - async getIdentity(id: string): Promise { - return withTx(ctx, "getIdentity", async (sql) => { - const [row] = await sql` - select id, email, name, created_at, updated_at - from ${sql.unsafe(schema)}.identity - where id = ${id} - `; - return row ? rowToIdentity(row) : null; - }); - }, - - async getIdentityByEmail(email: string): Promise { - return withTx(ctx, "getIdentityByEmail", async (sql) => { - const [row] = await sql` - select id, email, name, created_at, updated_at - from ${sql.unsafe(schema)}.identity - where email = ${email} - `; - return row ? rowToIdentity(row) : null; - }); - }, - - async updateIdentity( - id: string, - params: { name?: string; email?: string }, - ): Promise { - const { name, email } = params; - if (name === undefined && email === undefined) { - return false; - } - - return withTx(ctx, "updateIdentity", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.identity - set - ${name !== undefined ? sql`name = ${name},` : sql``} - ${email !== undefined ? sql`email = ${email},` : sql``} - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async deleteIdentity(id: string): Promise { - return withTx(ctx, "deleteIdentity", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.identity - where id = ${id} - `; - return result.count > 0; - }); - }, - }; -} - -export type IdentityOps = ReturnType; diff --git a/packages/accounts/ops/index.ts b/packages/accounts/ops/index.ts deleted file mode 100644 index 49f7fd8..0000000 --- a/packages/accounts/ops/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { deriveContext, setLocalAccountsTimeouts, withTx } from "./_tx"; -export { type DeviceAuthOps, deviceAuthOps } from "./device-auth"; -export { type EngineOps, engineOps } from "./engine"; -export { type IdentityOps, identityOps } from "./identity"; -export { type InvitationOps, invitationOps } from "./invitation"; -export { type OAuthOps, oauthOps } from "./oauth"; -export { type OrgOps, orgOps } from "./org"; -export { type OrgMemberOps, orgMemberOps } from "./org-member"; -export { type SessionOps, sessionOps } from "./session"; diff --git a/packages/accounts/ops/invitation.ts b/packages/accounts/ops/invitation.ts deleted file mode 100644 index 3526606..0000000 --- a/packages/accounts/ops/invitation.ts +++ /dev/null @@ -1,134 +0,0 @@ -import type { - AccountsContext, - CreateInvitationParams, - CreateInvitationResult, - Invitation, - OrgRole, -} from "../types"; -import { generateToken, tokenHash } from "../util/hash"; -import { withTx } from "./_tx"; - -interface InvitationRow { - id: string; - org_id: string; - email: string; - role: OrgRole; - invited_by: string; - expires_at: Date; - accepted_at: Date | null; - created_at: Date; -} - -function rowToInvitation(row: InvitationRow): Invitation { - return { - id: row.id, - orgId: row.org_id, - email: row.email, - role: row.role, - invitedBy: row.invited_by, - expiresAt: row.expires_at, - acceptedAt: row.accepted_at, - createdAt: row.created_at, - }; -} - -export function invitationOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createInvitation( - params: CreateInvitationParams, - ): Promise { - const { orgId, email, role, invitedBy, expiresInDays = 7 } = params; - - const rawToken = generateToken(); - const hash = tokenHash(rawToken); - const expiresAt = new Date(); - expiresAt.setDate(expiresAt.getDate() + expiresInDays); - - return withTx(ctx, "createInvitation", async (sql) => { - // Upsert: replace existing pending invitation for same org+email - const rows = await sql` - insert into ${sql.unsafe(schema)}.invitation - (org_id, email, role, token_hash, invited_by, expires_at) - values (${orgId}, ${email}, ${role}, ${hash}, ${invitedBy}, ${expiresAt}) - on conflict (org_id, email) - do update set - role = excluded.role, - token_hash = excluded.token_hash, - invited_by = excluded.invited_by, - expires_at = excluded.expires_at, - accepted_at = null - returning id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create invitation"); - } - return { - invitation: rowToInvitation(row), - rawToken, - }; - }); - }, - - async getInvitationByToken(rawToken: string): Promise { - const hash = tokenHash(rawToken); - - return withTx(ctx, "getInvitationByToken", async (sql) => { - // Single indexed lookup on the partial unique index: - // invitation_token_hash_uniq (token_hash) where accepted_at is null. - const rows = await sql` - select id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - from ${sql.unsafe(schema)}.invitation - where token_hash = ${hash} - and accepted_at is null - and expires_at > now() - limit 1 - `; - const row = rows[0]; - return row ? rowToInvitation(row) : null; - }); - }, - - async acceptInvitation(id: string): Promise { - return withTx(ctx, "acceptInvitation", async (sql) => { - const rows = await sql` - update ${sql.unsafe(schema)}.invitation - set accepted_at = now() - where id = ${id} - and accepted_at is null - returning id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - `; - const row = rows[0]; - return row ? rowToInvitation(row) : null; - }); - }, - - async revokeInvitation(id: string): Promise { - return withTx(ctx, "revokeInvitation", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.invitation - where id = ${id} - `; - return result.count > 0; - }); - }, - - async listPendingInvitations(orgId: string): Promise { - return withTx(ctx, "listPendingInvitations", async (sql) => { - const rows = await sql` - select id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - from ${sql.unsafe(schema)}.invitation - where org_id = ${orgId} - and accepted_at is null - and expires_at > now() - order by created_at - `; - return rows.map(rowToInvitation); - }); - }, - }; -} - -export type InvitationOps = ReturnType; diff --git a/packages/accounts/ops/oauth.ts b/packages/accounts/ops/oauth.ts deleted file mode 100644 index 177d2c3..0000000 --- a/packages/accounts/ops/oauth.ts +++ /dev/null @@ -1,101 +0,0 @@ -import type { - AccountsContext, - LinkOAuthParams, - OAuthAccount, - OAuthProvider, -} from "../types"; -import { withTx } from "./_tx"; - -interface OAuthAccountRow { - id: string; - identity_id: string; - provider: OAuthProvider; - provider_account_id: string; - email: string | null; - created_at: Date; - updated_at: Date | null; -} - -function rowToOAuthAccount(row: OAuthAccountRow): OAuthAccount { - return { - id: row.id, - identityId: row.identity_id, - provider: row.provider, - providerAccountId: row.provider_account_id, - email: row.email, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function oauthOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - // Login-only: we use the provider access token once during the OAuth - // callback to fetch the user's identity, then discard it. No provider - // tokens are persisted, so there is nothing to encrypt at rest. - async linkOAuthAccount(params: LinkOAuthParams): Promise { - const { identityId, provider, providerAccountId, email } = params; - - return withTx(ctx, "linkOAuthAccount", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.oauth_account - (identity_id, provider, provider_account_id, email) - values (${identityId}, ${provider}, ${providerAccountId}, ${email ?? null}) - on conflict (provider, provider_account_id) - do update set - identity_id = excluded.identity_id, - email = excluded.email, - updated_at = now() - returning id, identity_id, provider, provider_account_id, email, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to link OAuth account"); - } - return rowToOAuthAccount(row); - }); - }, - - async getOAuthAccount( - provider: OAuthProvider, - providerAccountId: string, - ): Promise { - return withTx(ctx, "getOAuthAccount", async (sql) => { - const [row] = await sql` - select id, identity_id, provider, provider_account_id, email, created_at, updated_at - from ${sql.unsafe(schema)}.oauth_account - where provider = ${provider} and provider_account_id = ${providerAccountId} - `; - return row ? rowToOAuthAccount(row) : null; - }); - }, - - async getOAuthAccountsByIdentity( - identityId: string, - ): Promise { - return withTx(ctx, "getOAuthAccountsByIdentity", async (sql) => { - const rows = await sql` - select id, identity_id, provider, provider_account_id, email, created_at, updated_at - from ${sql.unsafe(schema)}.oauth_account - where identity_id = ${identityId} - order by created_at - `; - return rows.map(rowToOAuthAccount); - }); - }, - - async unlinkOAuthAccount(id: string): Promise { - return withTx(ctx, "unlinkOAuthAccount", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.oauth_account - where id = ${id} - `; - return result.count > 0; - }); - }, - }; -} - -export type OAuthOps = ReturnType; diff --git a/packages/accounts/ops/org-member.ts b/packages/accounts/ops/org-member.ts deleted file mode 100644 index 9ea7e2b..0000000 --- a/packages/accounts/ops/org-member.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { - type AccountsContext, - AccountsError, - type OrgMember, - type OrgRole, -} from "../types"; -import { withTx } from "./_tx"; - -interface OrgMemberRow { - org_id: string; - identity_id: string; - role: OrgRole; - created_at: Date; - name: string; - email: string; -} - -function rowToOrgMember(row: OrgMemberRow): OrgMember { - return { - orgId: row.org_id, - identityId: row.identity_id, - role: row.role, - createdAt: row.created_at, - name: row.name, - email: row.email, - }; -} - -/** - * Verify that removing or demoting `identityId` from `orgId` would leave at - * least one other owner in place. Throws ORG_MUST_HAVE_OWNER otherwise. - * - * The query takes `for update` row locks on every other owner row in the - * org. This closes the race where two concurrent transactions, each - * removing a different owner, both observe the other's row as still - * present and proceed — leaving the org with zero owners. With `for - * update`, the second transaction blocks on the first's row locks, sees - * the post-commit count, and correctly fails. The previous DB trigger - * had the same race; this hardens it. - */ -async function assertAnotherOwnerExists( - sql: import("bun").SQL, - schema: string, - orgId: string, - identityId: string, -): Promise { - const rows = await sql<{ identity_id: string }[]>` - select identity_id - from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} - and role = 'owner' - and identity_id <> ${identityId} - for update - `; - if (rows.length === 0) { - throw new AccountsError( - "ORG_MUST_HAVE_OWNER", - "Cannot remove the last owner from an organization", - ); - } -} - -export function orgMemberOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async addMember( - orgId: string, - identityId: string, - role: OrgRole, - ): Promise { - return withTx(ctx, "addMember", async (sql) => { - const rows = await sql` - with inserted as ( - insert into ${sql.unsafe(schema)}.org_member (org_id, identity_id, role) - values (${orgId}, ${identityId}, ${role}) - returning org_id, identity_id, role, created_at - ) - select i.*, id.name, id.email - from inserted i - join ${sql.unsafe(schema)}.identity id on id.id = i.identity_id - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to add member"); - } - return rowToOrgMember(row); - }); - }, - - async removeMember(orgId: string, identityId: string): Promise { - return withTx(ctx, "removeMember", async (sql) => { - // Fetch the target row with `for update` so the row stays stable - // under our feet while we decide whether to delete it. - const [target] = await sql<{ role: OrgRole }[]>` - select role - from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} and identity_id = ${identityId} - for update - `; - if (!target) return false; - - if (target.role === "owner") { - await assertAnotherOwnerExists(sql, schema, orgId, identityId); - } - - const result = await sql` - delete from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} and identity_id = ${identityId} - `; - return result.count > 0; - }); - }, - - async updateRole( - orgId: string, - identityId: string, - newRole: OrgRole, - ): Promise { - return withTx(ctx, "updateRole", async (sql) => { - const [target] = await sql<{ role: OrgRole }[]>` - select role - from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} and identity_id = ${identityId} - for update - `; - if (!target) return false; - - // Only block the transition that would orphan the org: an owner - // being changed to anything other than owner. - if (target.role === "owner" && newRole !== "owner") { - await assertAnotherOwnerExists(sql, schema, orgId, identityId); - } - - const result = await sql` - update ${sql.unsafe(schema)}.org_member - set role = ${newRole} - where org_id = ${orgId} and identity_id = ${identityId} - `; - return result.count > 0; - }); - }, - - async getMember( - orgId: string, - identityId: string, - ): Promise { - return withTx(ctx, "getMember", async (sql) => { - const [row] = await sql` - select m.org_id, m.identity_id, m.role, m.created_at, id.name, id.email - from ${sql.unsafe(schema)}.org_member m - join ${sql.unsafe(schema)}.identity id on id.id = m.identity_id - where m.org_id = ${orgId} and m.identity_id = ${identityId} - `; - return row ? rowToOrgMember(row) : null; - }); - }, - - async listMembers(orgId: string): Promise { - return withTx(ctx, "listMembers", async (sql) => { - const rows = await sql` - select m.org_id, m.identity_id, m.role, m.created_at, id.name, id.email - from ${sql.unsafe(schema)}.org_member m - join ${sql.unsafe(schema)}.identity id on id.id = m.identity_id - where m.org_id = ${orgId} - order by m.created_at - `; - return rows.map(rowToOrgMember); - }); - }, - - async countOwnedOrgs(identityId: string): Promise { - return withTx(ctx, "countOwnedOrgs", async (sql) => { - const [row] = await sql<{ count: number }[]>` - select count(*)::int as count - from ${sql.unsafe(schema)}.org_member - where identity_id = ${identityId} and role = 'owner' - `; - return row?.count ?? 0; - }); - }, - - async listOwners(orgId: string): Promise { - return withTx(ctx, "listOwners", async (sql) => { - const rows = await sql` - select m.org_id, m.identity_id, m.role, m.created_at, id.name, id.email - from ${sql.unsafe(schema)}.org_member m - join ${sql.unsafe(schema)}.identity id on id.id = m.identity_id - where m.org_id = ${orgId} and m.role = 'owner' - order by m.created_at - `; - return rows.map(rowToOrgMember); - }); - }, - }; -} - -export type OrgMemberOps = ReturnType; diff --git a/packages/accounts/ops/org.ts b/packages/accounts/ops/org.ts deleted file mode 100644 index 96f828b..0000000 --- a/packages/accounts/ops/org.ts +++ /dev/null @@ -1,110 +0,0 @@ -import type { AccountsContext, CreateOrgParams, Org } from "../types"; -import { generateSlug } from "../util/slug"; -import { withTx } from "./_tx"; - -interface OrgRow { - id: string; - slug: string; - name: string; - created_at: Date; - updated_at: Date | null; -} - -function rowToOrg(row: OrgRow): Org { - return { - id: row.id, - slug: row.slug, - name: row.name, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function orgOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createOrg(params: CreateOrgParams): Promise { - const { id, name } = params; - const slug = generateSlug(); - - return withTx(ctx, "createOrg", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.org (id, slug, name) - values (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${slug}, ${name}) - returning id, slug, name, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create org"); - } - return rowToOrg(row); - }); - }, - - async getOrg(id: string): Promise { - return withTx(ctx, "getOrg", async (sql) => { - const [row] = await sql` - select id, slug, name, created_at, updated_at - from ${sql.unsafe(schema)}.org - where id = ${id} - `; - return row ? rowToOrg(row) : null; - }); - }, - - async getOrgBySlug(slug: string): Promise { - return withTx(ctx, "getOrgBySlug", async (sql) => { - const [row] = await sql` - select id, slug, name, created_at, updated_at - from ${sql.unsafe(schema)}.org - where slug = ${slug} - `; - return row ? rowToOrg(row) : null; - }); - }, - - async updateOrg(id: string, params: { name?: string }): Promise { - const { name } = params; - if (name === undefined) { - return false; - } - - return withTx(ctx, "updateOrg", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.org - set - name = ${name}, - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async deleteOrg(id: string): Promise { - return withTx(ctx, "deleteOrg", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.org - where id = ${id} - `; - return result.count > 0; - }); - }, - - async listOrgsByIdentity(identityId: string): Promise { - return withTx(ctx, "listOrgsByIdentity", async (sql) => { - const rows = await sql` - select o.id, o.slug, o.name, o.created_at, o.updated_at - from ${sql.unsafe(schema)}.org o - inner join ${sql.unsafe(schema)}.org_member m on m.org_id = o.id - where m.identity_id = ${identityId} - order by o.created_at - `; - return rows.map(rowToOrg); - }); - }, - }; -} - -export type OrgOps = ReturnType; diff --git a/packages/accounts/ops/session.ts b/packages/accounts/ops/session.ts deleted file mode 100644 index ed4119b..0000000 --- a/packages/accounts/ops/session.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { - AccountsContext, - CreateSessionParams, - CreateSessionResult, - Identity, - Session, -} from "../types"; -import { generateToken, tokenHash } from "../util/hash"; -import { withTx } from "./_tx"; - -interface SessionRow { - id: string; - identity_id: string; - expires_at: Date; - created_at: Date; -} - -interface SessionWithIdentityRow extends SessionRow { - identity_email: string; - identity_name: string; - identity_created_at: Date; - identity_updated_at: Date | null; -} - -function rowToSession(row: SessionRow): Session { - return { - id: row.id, - identityId: row.identity_id, - expiresAt: row.expires_at, - createdAt: row.created_at, - }; -} - -export function sessionOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createSession( - params: CreateSessionParams, - ): Promise { - const { identityId, expiresInDays = 30 } = params; - - const rawToken = generateToken(); - const hash = tokenHash(rawToken); - const expiresAt = new Date(); - expiresAt.setDate(expiresAt.getDate() + expiresInDays); - - return withTx(ctx, "createSession", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.session (identity_id, token_hash, expires_at) - values (${identityId}, ${hash}, ${expiresAt}) - returning id, identity_id, expires_at, created_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create session"); - } - return { - session: rowToSession(row), - rawToken, - }; - }); - }, - - async validateSession( - rawToken: string, - ): Promise<{ session: Session; identity: Identity } | null> { - const hash = tokenHash(rawToken); - - return withTx(ctx, "validateSession", async (sql) => { - // Single indexed lookup: token_hash is unique. No loop, no argon2. - const rows = await sql` - select - s.id, s.identity_id, s.expires_at, s.created_at, - i.email as identity_email, i.name as identity_name, - i.created_at as identity_created_at, i.updated_at as identity_updated_at - from ${sql.unsafe(schema)}.session s - inner join ${sql.unsafe(schema)}.identity i on i.id = s.identity_id - where s.token_hash = ${hash} - and s.expires_at > now() - limit 1 - `; - const row = rows[0]; - if (!row) { - return null; - } - return { - session: rowToSession(row), - identity: { - id: row.identity_id, - email: row.identity_email, - name: row.identity_name, - createdAt: row.identity_created_at, - updatedAt: row.identity_updated_at, - }, - }; - }); - }, - - async deleteSession(id: string): Promise { - return withTx(ctx, "deleteSession", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.session - where id = ${id} - `; - return result.count > 0; - }); - }, - - async deleteSessionsByIdentity(identityId: string): Promise { - return withTx(ctx, "deleteSessionsByIdentity", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.session - where identity_id = ${identityId} - `; - return result.count; - }); - }, - - async cleanupExpiredSessions(): Promise { - return withTx(ctx, "cleanupExpiredSessions", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.session - where expires_at <= now() - `; - return result.count; - }); - }, - }; -} - -export type SessionOps = ReturnType; diff --git a/packages/accounts/package.json b/packages/accounts/package.json deleted file mode 100644 index 0bab4d5..0000000 --- a/packages/accounts/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "@memory.build/accounts", - "version": "0.2.5", - "private": true, - "type": "module", - "dependencies": { - "@pydantic/logfire-node": "^0.13.1" - } -} diff --git a/packages/accounts/types.ts b/packages/accounts/types.ts deleted file mode 100644 index 764d300..0000000 --- a/packages/accounts/types.ts +++ /dev/null @@ -1,213 +0,0 @@ -import type { SQL } from "bun"; - -// ============================================================================= -// Context -// ============================================================================= - -export interface AccountsContext { - sql: SQL; - schema: string; - inTransaction: boolean; -} - -// ============================================================================= -// Errors -// ============================================================================= - -export type AccountsErrorCode = - | "ORG_MUST_HAVE_OWNER" - | "IDENTITY_NOT_FOUND" - | "ORG_NOT_FOUND" - | "ENGINE_NOT_FOUND" - | "INVITATION_NOT_FOUND" - | "INVITATION_EXPIRED" - | "INVITATION_ALREADY_ACCEPTED" - | "SESSION_NOT_FOUND" - | "SESSION_EXPIRED" - | "OAUTH_ACCOUNT_NOT_FOUND" - | "DUPLICATE_SLUG" - | "DUPLICATE_EMAIL"; - -export class AccountsError extends Error { - constructor( - public code: AccountsErrorCode, - message: string, - ) { - super(message); - this.name = "AccountsError"; - } -} - -// ============================================================================= -// Identity -// ============================================================================= - -export interface Identity { - id: string; - email: string; - name: string; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateIdentityParams { - id?: string; - email: string; - name: string; -} - -// ============================================================================= -// Org -// ============================================================================= - -export interface Org { - id: string; - slug: string; - name: string; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateOrgParams { - id?: string; - name: string; -} - -// ============================================================================= -// OrgMember -// ============================================================================= - -export type OrgRole = "owner" | "admin" | "member"; - -export interface OrgMember { - orgId: string; - identityId: string; - role: OrgRole; - createdAt: Date; - name: string; - email: string; -} - -// ============================================================================= -// Engine -// ============================================================================= - -export type EngineStatus = "active" | "suspended" | "deleted"; - -export interface Engine { - id: string; - orgId: string; - slug: string; - name: string; - shardId: number; - status: EngineStatus; - language: string; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateEngineParams { - id?: string; - orgId: string; - name: string; - shardId?: number; - language?: string; // defaults to 'english' -} - -// ============================================================================= -// Invitation -// ============================================================================= - -export interface Invitation { - id: string; - orgId: string; - email: string; - role: OrgRole; - invitedBy: string; - expiresAt: Date; - acceptedAt: Date | null; - createdAt: Date; -} - -export interface CreateInvitationParams { - orgId: string; - email: string; - role: OrgRole; - invitedBy: string; - expiresInDays?: number; -} - -export interface CreateInvitationResult { - invitation: Invitation; - rawToken: string; -} - -// ============================================================================= -// OAuthAccount -// ============================================================================= - -export type OAuthProvider = "google" | "github"; - -export interface OAuthAccount { - id: string; - identityId: string; - provider: OAuthProvider; - providerAccountId: string; - email: string | null; - createdAt: Date; - updatedAt: Date | null; -} - -export interface LinkOAuthParams { - identityId: string; - provider: OAuthProvider; - providerAccountId: string; - email?: string; -} - -// ============================================================================= -// Session -// ============================================================================= - -export interface Session { - id: string; - identityId: string; - expiresAt: Date; - createdAt: Date; -} - -export interface CreateSessionParams { - identityId: string; - expiresInDays?: number; -} - -export interface CreateSessionResult { - session: Session; - rawToken: string; -} - -// ============================================================================= -// DeviceAuthorization (OAuth Device Flow) -// ============================================================================= - -export type DeviceProvider = "google" | "github"; - -export interface DeviceAuthorization { - deviceCode: string; - userCode: string; - provider: DeviceProvider; - oauthState: string; - expiresAt: Date; - lastPoll: Date | null; - identityId: string | null; - denied: boolean; - createdAt: Date; -} - -export interface CreateDeviceAuthParams { - deviceCode: string; - userCode: string; - provider: DeviceProvider; - oauthState: string; - expiresAt: Date; -} diff --git a/packages/accounts/util/hash.test.ts b/packages/accounts/util/hash.test.ts deleted file mode 100644 index 0b37721..0000000 --- a/packages/accounts/util/hash.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { generateToken, tokenHash } from "./hash"; - -describe("generateToken", () => { - test("produces a base64url string of the expected length", () => { - const token = generateToken(); - // 32 bytes base64url-encoded with padding stripped: ceil(32 / 3) * 4 = 44, - // minus the trailing '=' padding character → 43 chars. - expect(token).toHaveLength(43); - expect(token).toMatch(/^[A-Za-z0-9_-]+$/); - }); - - test("produces unique values across calls", () => { - const seen = new Set(); - for (let i = 0; i < 100; i++) { - seen.add(generateToken()); - } - expect(seen.size).toBe(100); - }); -}); - -describe("tokenHash", () => { - test("returns 32 raw bytes", () => { - const hash = tokenHash("anything"); - expect(hash).toBeInstanceOf(Buffer); - expect(hash.length).toBe(32); - }); - - test("is deterministic", () => { - const a = tokenHash("hello world"); - const b = tokenHash("hello world"); - expect(a.equals(b)).toBe(true); - }); - - test("differs for different inputs", () => { - const a = tokenHash("hello world"); - const b = tokenHash("hello world!"); - expect(a.equals(b)).toBe(false); - }); - - test("matches the published sha256 of a known string", () => { - // Known value: sha256("abc") = ba7816bf8f01cfea414140de5dae2223 - // b00361a396177a9cb410ff61f20015ad - const hex = tokenHash("abc").toString("hex"); - expect(hex).toBe( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - ); - }); -}); diff --git a/packages/accounts/util/hash.ts b/packages/accounts/util/hash.ts deleted file mode 100644 index 37c9ccf..0000000 --- a/packages/accounts/util/hash.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Token utilities for session and invitation tokens. - * - * Tokens are 256-bit CSPRNG output (base64url-encoded). We store sha256(token) - * in a unique-indexed column and look up by it directly — no slow-hash verifier - * is needed because the token's entropy alone defeats offline preimage attacks. - * See migration 009_session_lookup.sql for the rationale. - */ - -/** - * Generate a random token (32 bytes, base64url encoded). - */ -export function generateToken(): string { - const bytes = crypto.getRandomValues(new Uint8Array(32)); - return btoa(String.fromCharCode(...bytes)) - .replace(/\+/g, "-") - .replace(/\//g, "_") - .replace(/=/g, ""); -} - -/** - * Compute the lookup hash stored in the session/invitation `token_hash` column. - * Returns 32 raw bytes suitable for binding directly to a `bytea` parameter. - */ -export function tokenHash(rawToken: string): Buffer { - return new Bun.CryptoHasher("sha256").update(rawToken).digest(); -} diff --git a/packages/accounts/util/slug.ts b/packages/accounts/util/slug.ts deleted file mode 100644 index 4bb81b5..0000000 --- a/packages/accounts/util/slug.ts +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Slug generation - * - * Generates a 12-character alphanumeric slug (a-z, 0-9) for org and engine identification. - */ - -const SLUG_LENGTH = 12; -const SLUG_CHARSET = "abcdefghijklmnopqrstuvwxyz0123456789"; - -/** - * Generate a random slug (12 lowercase alphanumeric chars) - */ -export function generateSlug(): string { - const bytes = crypto.getRandomValues(new Uint8Array(SLUG_LENGTH)); - let result = ""; - for (const byte of bytes) { - result += SLUG_CHARSET[byte % SLUG_CHARSET.length]; - } - return result; -} diff --git a/packages/server/package.json b/packages/server/package.json index cab7bba..d4a9840 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -2,7 +2,6 @@ "name": "memory-engine-server", "version": "0.2.5", "dependencies": { - "@memory.build/accounts": "workspace:*", "@memory.build/auth": "workspace:*", "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", diff --git a/scripts/release-server.ts b/scripts/release-server.ts index 1096d30..41e8bdf 100644 --- a/scripts/release-server.ts +++ b/scripts/release-server.ts @@ -25,11 +25,10 @@ import semver from "semver"; const root = join(import.meta.dirname, ".."); // Server-side package.json files. `packages/server/package.json` is the -// canonical source of truth for SERVER_VERSION; the other four are sibling +// canonical source of truth for SERVER_VERSION; the others are sibling // internal packages consumed by the server via `workspace:*`. They are // bumped in lockstep for visual consistency — none of them publish. const PACKAGE_JSONS = [ - "packages/accounts/package.json", "packages/embedding/package.json", "packages/engine/package.json", "packages/server/package.json", From 963e495c5aee5b317725f7fa9ea4c497f25986c1 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 21:29:56 +0200 Subject: [PATCH 062/156] feat(engine): trim packages/engine to core + space (5C) The engine package now holds only the new-model runtime layers. Delete the legacy ops/db/migrate/types/util (the old createEngineDB, RLS me_ ops, per-engine migration runner, flat type + api-key util). The root index re-exports just the core + space namespaces; subpath imports are unchanged. Repoint the five remaining `@memory.build/engine` root importers (provision + the server integration tests) to the /core and /space subpaths. Test parity: the deleted util/api-key.test.ts is replaced by a new core/api-key.test.ts covering parseApiKey/formatApiKey round-trip, generateLookupId/generateSecret, and hashApiKeySecret (core had no api-key unit test before). Legacy ops/migrate tests are superseded by core + space integration suites. typecheck + lint + unit (656) + engine/server integration (90) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/engine/core/api-key.test.ts | 102 ++ packages/engine/db.integration.test.ts | 1010 ----------------- packages/engine/db.ts | 183 --- packages/engine/index.ts | 50 +- packages/engine/migrate/bootstrap.ts | 99 -- packages/engine/migrate/discover.test.ts | 106 -- packages/engine/migrate/discover.ts | 48 - packages/engine/migrate/index.ts | 21 - .../migrate/migrate.integration.test.ts | 823 -------------- .../migrate/migrations/001_updated_at.sql | 10 - .../engine/migrate/migrations/002_memory.sql | 57 - .../migrate/migrations/003_memory_trigger.sql | 34 - .../migrate/migrations/004_auth_tables.sql | 227 ---- .../migrations/005_embedding_queue.sql | 150 --- .../migrations/006_prune_embedding_queue.sql | 27 - .../007_drop_api_key_last_used_at.sql | 2 - packages/engine/migrate/migrations/sql.d.ts | 4 - packages/engine/migrate/provision.ts | 64 -- packages/engine/migrate/runner.test.ts | 31 - packages/engine/migrate/runner.ts | 283 ----- packages/engine/migrate/template.test.ts | 185 --- packages/engine/migrate/template.ts | 38 - packages/engine/migrate/test-utils.ts | 159 --- packages/engine/ops/_tx.ts | 134 --- packages/engine/ops/api-key.ts | 195 ---- packages/engine/ops/grant.ts | 172 --- packages/engine/ops/index.ts | 7 - packages/engine/ops/memory.test.ts | 127 --- packages/engine/ops/memory.ts | 778 ------------- packages/engine/ops/owner.ts | 128 --- packages/engine/ops/role.ts | 142 --- packages/engine/ops/user.ts | 193 ---- packages/engine/types.ts | 258 ----- packages/engine/util/api-key.test.ts | 139 --- packages/engine/util/api-key.ts | 125 -- packages/engine/util/index.ts | 9 - .../authenticate-space.integration.test.ts | 2 +- packages/server/provision.integration.test.ts | 2 +- packages/server/provision.ts | 2 +- .../rpc/memory/management.integration.test.ts | 3 +- .../rpc/memory/memory.integration.test.ts | 3 +- 41 files changed, 115 insertions(+), 6017 deletions(-) create mode 100644 packages/engine/core/api-key.test.ts delete mode 100644 packages/engine/db.integration.test.ts delete mode 100644 packages/engine/db.ts delete mode 100644 packages/engine/migrate/bootstrap.ts delete mode 100644 packages/engine/migrate/discover.test.ts delete mode 100644 packages/engine/migrate/discover.ts delete mode 100644 packages/engine/migrate/index.ts delete mode 100644 packages/engine/migrate/migrate.integration.test.ts delete mode 100644 packages/engine/migrate/migrations/001_updated_at.sql delete mode 100644 packages/engine/migrate/migrations/002_memory.sql delete mode 100644 packages/engine/migrate/migrations/003_memory_trigger.sql delete mode 100644 packages/engine/migrate/migrations/004_auth_tables.sql delete mode 100644 packages/engine/migrate/migrations/005_embedding_queue.sql delete mode 100644 packages/engine/migrate/migrations/006_prune_embedding_queue.sql delete mode 100644 packages/engine/migrate/migrations/007_drop_api_key_last_used_at.sql delete mode 100644 packages/engine/migrate/migrations/sql.d.ts delete mode 100644 packages/engine/migrate/provision.ts delete mode 100644 packages/engine/migrate/runner.test.ts delete mode 100644 packages/engine/migrate/runner.ts delete mode 100644 packages/engine/migrate/template.test.ts delete mode 100644 packages/engine/migrate/template.ts delete mode 100644 packages/engine/migrate/test-utils.ts delete mode 100644 packages/engine/ops/_tx.ts delete mode 100644 packages/engine/ops/api-key.ts delete mode 100644 packages/engine/ops/grant.ts delete mode 100644 packages/engine/ops/index.ts delete mode 100644 packages/engine/ops/memory.test.ts delete mode 100644 packages/engine/ops/memory.ts delete mode 100644 packages/engine/ops/owner.ts delete mode 100644 packages/engine/ops/role.ts delete mode 100644 packages/engine/ops/user.ts delete mode 100644 packages/engine/types.ts delete mode 100644 packages/engine/util/api-key.test.ts delete mode 100644 packages/engine/util/api-key.ts delete mode 100644 packages/engine/util/index.ts diff --git a/packages/engine/core/api-key.test.ts b/packages/engine/core/api-key.test.ts new file mode 100644 index 0000000..2dccdd6 --- /dev/null +++ b/packages/engine/core/api-key.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, test } from "bun:test"; +import { + formatApiKey, + generateLookupId, + generateSecret, + hashApiKeySecret, + parseApiKey, +} from "./api-key"; + +describe("generateLookupId", () => { + test("generates a 16-char string", () => { + expect(generateLookupId()).toHaveLength(16); + }); + + test("only contains valid lookup_id characters", () => { + expect(generateLookupId()).toMatch(/^[A-Za-z0-9_-]{16}$/); + }); + + test("generates unique values", () => { + expect(generateLookupId()).not.toBe(generateLookupId()); + }); +}); + +describe("generateSecret", () => { + test("generates a 32-char string", () => { + expect(generateSecret()).toHaveLength(32); + }); + + test("only contains base64url characters", () => { + expect(generateSecret()).toMatch(/^[A-Za-z0-9_-]{32}$/); + }); + + test("generates unique values", () => { + expect(generateSecret()).not.toBe(generateSecret()); + }); +}); + +describe("hashApiKeySecret", () => { + test("is a stable hex sha256 digest", () => { + const h = hashApiKeySecret("a-secret"); + expect(h).toMatch(/^[0-9a-f]{64}$/); + expect(hashApiKeySecret("a-secret")).toBe(h); + }); + + test("different secrets produce different hashes", () => { + expect(hashApiKeySecret("secret-a")).not.toBe(hashApiKeySecret("secret-b")); + }); +}); + +describe("formatApiKey", () => { + test("formats key with all parts", () => { + expect( + formatApiKey("abc123def456", "lookupid12345678", "s".repeat(32)), + ).toBe(`me.abc123def456.lookupid12345678.${"s".repeat(32)}`); + }); +}); + +describe("parseApiKey", () => { + const valid = `me.abc123def456.lookupid12345678.${"s".repeat(32)}`; + + test("parses a valid key (round-trips with formatApiKey)", () => { + const parsed = parseApiKey(valid); + expect(parsed).toEqual({ + spaceSlug: "abc123def456", + lookupId: "lookupid12345678", + secret: "s".repeat(32), + }); + if (parsed) { + expect( + formatApiKey(parsed.spaceSlug, parsed.lookupId, parsed.secret), + ).toBe(valid); + } + }); + + test("returns null for the wrong prefix", () => { + expect( + parseApiKey(`x.abc123def456.lookupid12345678.${"s".repeat(32)}`), + ).toBeNull(); + }); + + test("returns null for an invalid spaceSlug (uppercase)", () => { + expect( + parseApiKey(`me.ABC123def456.lookupid12345678.${"s".repeat(32)}`), + ).toBeNull(); + }); + + test("returns null for a short spaceSlug", () => { + expect(parseApiKey(`me.abc.lookupid12345678.${"s".repeat(32)}`)).toBeNull(); + }); + + test("returns null for an invalid lookupId", () => { + expect(parseApiKey(`me.abc123def456.short.${"s".repeat(32)}`)).toBeNull(); + }); + + test("returns null for the wrong secret length", () => { + expect(parseApiKey("me.abc123def456.lookupid12345678.tooshort")).toBeNull(); + }); + + test("returns null for the wrong number of parts", () => { + expect(parseApiKey("me.abc123def456.lookupid12345678")).toBeNull(); + }); +}); diff --git a/packages/engine/db.integration.test.ts b/packages/engine/db.integration.test.ts deleted file mode 100644 index 39eb37b..0000000 --- a/packages/engine/db.integration.test.ts +++ /dev/null @@ -1,1010 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { SQL } from "bun"; -import { createEngineDB } from "./db"; -import { bootstrap } from "./migrate/bootstrap"; -import { provisionEngine } from "./migrate/provision"; -import { TestDatabase } from "./migrate/test-utils"; - -const testDb = new TestDatabase(); -let connectionString: string; -let sql: SQL; -const schema = "me_testengine01"; - -beforeAll(async () => { - connectionString = await testDb.create(); - sql = new SQL(connectionString); - await bootstrap(sql); - await provisionEngine(sql, "testengine01", undefined, "0.1.0"); -}); - -afterAll(async () => { - await sql.close(); - await testDb.drop(); -}); - -// --------------------------------------------------------------------------- -// Principal Tests -// --------------------------------------------------------------------------- -describe("user ops", () => { - test("createUser creates a principal", async () => { - const db = createEngineDB(sql, schema); - const user = await db.createUser({ - name: "test-user", - }); - - expect(user.name).toBe("test-user"); - expect(user.superuser).toBe(false); - expect(user.id).toBeDefined(); - expect(user.createdAt).toBeInstanceOf(Date); - }); - - test("createUser with custom id", async () => { - const db = createEngineDB(sql, schema); - // Generate a UUIDv7 for testing - const customId = crypto.randomUUID(); - // Replace version nibble with 7 to make it UUIDv7 compatible - const uuidv7Id = customId.replace( - /^(.{8}-.{4}-)(.)/, - (_, prefix) => `${prefix}7`, - ); - - const user = await db.createUser({ - id: uuidv7Id, - name: "test-user-with-id", - }); - - expect(user.id).toBe(uuidv7Id); - expect(user.name).toBe("test-user-with-id"); - }); - - test("createSuperuser creates a superuser principal", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("admin"); - - expect(superuser.name).toBe("admin"); - expect(superuser.superuser).toBe(true); - }); - - test("createSuperuser with custom id", async () => { - const db = createEngineDB(sql, schema); - const customId = crypto - .randomUUID() - .replace(/^(.{8}-.{4}-)(.)/, (_, prefix) => `${prefix}7`); - const superuser = await db.createSuperuser("admin-with-id", customId); - - expect(superuser.id).toBe(customId); - expect(superuser.name).toBe("admin-with-id"); - expect(superuser.superuser).toBe(true); - }); - - test("createUser with identityId and canLogin", async () => { - const db = createEngineDB(sql, schema); - const identityId = crypto - .randomUUID() - .replace(/^(.{8}-.{4}-)(.)/, (_, prefix) => `${prefix}7`); - - const user = await db.createUser({ - name: "owned-user", - identityId: identityId, - canLogin: true, - }); - - expect(user.name).toBe("owned-user"); - expect(user.identityId).toBe(identityId); - expect(user.canLogin).toBe(true); - }); - - test("createRole creates a user with canLogin=false", async () => { - const db = createEngineDB(sql, schema); - const role = await db.createRole("test-role"); - - expect(role.name).toBe("test-role"); - expect(role.canLogin).toBe(false); - expect(role.superuser).toBe(false); - }); - - test("getUser returns user by ID", async () => { - const db = createEngineDB(sql, schema); - const created = await db.createUser({ name: "get-by-id-test" }); - const fetched = await db.getUser(created.id); - - expect(fetched).not.toBeNull(); - expect(fetched!.id).toBe(created.id); - expect(fetched!.name).toBe("get-by-id-test"); - }); - - test("getUser returns null for non-existent ID", async () => { - const db = createEngineDB(sql, schema); - const fetched = await db.getUser("00000000-0000-0000-0000-000000000000"); - - expect(fetched).toBeNull(); - }); - - test("getUserByName returns principal by name", async () => { - const db = createEngineDB(sql, schema); - await db.createUser({ name: "get-by-name-test" }); - const fetched = await db.getUserByName("get-by-name-test"); - - expect(fetched).not.toBeNull(); - expect(fetched!.name).toBe("get-by-name-test"); - }); - - test("getUserByName matches case-insensitively (citext)", async () => { - const db = createEngineDB(sql, schema); - const uniqueName = `ExactMatch_${Date.now()}`; - await db.createUser({ name: uniqueName }); - const fetched = await db.getUserByName(uniqueName); - - expect(fetched).not.toBeNull(); - expect(fetched!.name).toBe(uniqueName); - - // citext: different case should still match - const alsoFound = await db.getUserByName(uniqueName.toLowerCase()); - expect(alsoFound).not.toBeNull(); - expect(alsoFound!.id).toBe(fetched!.id); - }); - - test("listUsers returns all principals", async () => { - const db = createEngineDB(sql, schema); - const principals = await db.listUsers(); - - expect(principals.length).toBeGreaterThan(0); - expect(principals[0]!.id).toBeDefined(); - }); - - test("renameUser updates name", async () => { - const db = createEngineDB(sql, schema); - const created = await db.createUser({ name: "rename-test" }); - const result = await db.renameUser(created.id, "renamed-test"); - - expect(result).toBe(true); - - const fetched = await db.getUser(created.id); - expect(fetched!.name).toBe("renamed-test"); - }); - - test("deleteUser removes principal", async () => { - const db = createEngineDB(sql, schema); - const created = await db.createUser({ name: "delete-test" }); - const result = await db.deleteUser(created.id); - - expect(result).toBe(true); - - const fetched = await db.getUser(created.id); - expect(fetched).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// Grant Tests -// --------------------------------------------------------------------------- -describe("grant ops", () => { - let testPrincipalId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - const user = await db.createUser({ name: "grant-test-user" }); - testPrincipalId = user.id; - }); - - test("grantTreeAccess creates a grant", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "test.path", - actions: ["read", "create"], - }); - - const grant = await db.getTreeGrant(testPrincipalId, "test.path"); - expect(grant).not.toBeNull(); - expect(grant!.userId).toBe(testPrincipalId); - expect(grant!.treePath).toBe("test.path"); - expect(grant!.actions).toContain("read"); - expect(grant!.actions).toContain("create"); - }); - - test("grantTreeAccess upserts on conflict", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "upsert.path", - actions: ["read"], - }); - - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "upsert.path", - actions: ["read", "create", "update"], - }); - - const grant = await db.getTreeGrant(testPrincipalId, "upsert.path"); - expect(grant!.actions).toHaveLength(3); - }); - - test("revokeTreeAccess removes grant", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "revoke.path", - actions: ["read"], - }); - - const result = await db.revokeTreeAccess(testPrincipalId, "revoke.path"); - expect(result).toBe(true); - - const grant = await db.getTreeGrant(testPrincipalId, "revoke.path"); - expect(grant).toBeNull(); - }); - - test("listTreeGrants returns grants for principal", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "list.path", - actions: ["read"], - }); - - const grants = await db.listTreeGrants(testPrincipalId); - expect(grants.length).toBeGreaterThan(0); - }); - - test("checkTreeAccess uses has_tree_access function", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("access-check-admin"); - - // Superuser should have access to everything - const hasAccess = await db.checkTreeAccess( - superuser.id, - "any.path", - "read", - ); - expect(hasAccess).toBe(true); - }); -}); - -// --------------------------------------------------------------------------- -// Owner Tests -// --------------------------------------------------------------------------- -describe("owner ops", () => { - let testPrincipalId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - const user = await db.createUser({ name: "owner-test-user" }); - testPrincipalId = user.id; - }); - - test("setTreeOwner creates ownership", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "owned.path"); - - const owner = await db.getTreeOwner("owned.path"); - expect(owner).not.toBeNull(); - expect(owner!.userId).toBe(testPrincipalId); - expect(owner!.treePath).toBe("owned.path"); - }); - - test("setTreeOwner upserts on conflict", async () => { - const db = createEngineDB(sql, schema); - const otherPrincipal = await db.createUser({ name: "other-owner" }); - - await db.setTreeOwner(testPrincipalId, "upsert.owned"); - await db.setTreeOwner(otherPrincipal.id, "upsert.owned"); - - const owner = await db.getTreeOwner("upsert.owned"); - expect(owner!.userId).toBe(otherPrincipal.id); - }); - - test("removeTreeOwner removes ownership", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "remove.owned"); - const result = await db.removeTreeOwner("remove.owned"); - - expect(result).toBe(true); - - const owner = await db.getTreeOwner("remove.owned"); - expect(owner).toBeNull(); - }); - - test("listTreeOwners returns owners for principal", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "list.owned"); - - const owners = await db.listTreeOwners(testPrincipalId); - expect(owners.length).toBeGreaterThan(0); - }); - - test("isOwnerOf checks ownership", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "isowner.path"); - - const isOwner = await db.isOwnerOf(testPrincipalId, "isowner.path.child"); - expect(isOwner).toBe(true); - - const isNotOwner = await db.isOwnerOf(testPrincipalId, "other.path"); - expect(isNotOwner).toBe(false); - }); -}); - -// --------------------------------------------------------------------------- -// Role Tests -// --------------------------------------------------------------------------- -describe("role ops", () => { - let roleId: string; - let memberId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - // Create a role (a principal used as a role for grouping) - const role = await db.createUser({ - name: "membership-role", - }); - roleId = role.id; - - const member = await db.createUser({ name: "role-member" }); - memberId = member.id; - }); - - test("addRoleMember adds member to role", async () => { - const db = createEngineDB(sql, schema); - await db.addRoleMember(roleId, memberId); - - const members = await db.listRoleMembers(roleId); - expect(members.length).toBeGreaterThan(0); - expect(members.some((m) => m.memberId === memberId)).toBe(true); - }); - - test("addRoleMember detects cycles", async () => { - const db = createEngineDB(sql, schema); - const role1 = await db.createUser({ - name: "cycle-role-1", - }); - const role2 = await db.createUser({ - name: "cycle-role-2", - }); - - await db.addRoleMember(role1.id, role2.id); - - // Try to create cycle: role2 -> role1 (but role1 -> role2 exists) - await expect(db.addRoleMember(role2.id, role1.id)).rejects.toThrow( - "would create a cycle", - ); - }); - - test("removeRoleMember removes member from role", async () => { - const db = createEngineDB(sql, schema); - const role = await db.createUser({ - name: "remove-role", - }); - const member = await db.createUser({ name: "remove-member" }); - - await db.addRoleMember(role.id, member.id); - const result = await db.removeRoleMember(role.id, member.id); - - expect(result).toBe(true); - - const members = await db.listRoleMembers(role.id); - expect(members.some((m) => m.memberId === member.id)).toBe(false); - }); - - test("listRolesForUser returns roles", async () => { - const db = createEngineDB(sql, schema); - const roles = await db.listRolesForUser(memberId); - - expect(roles.some((r) => r.id === roleId)).toBe(true); - }); - - test("hasAdminOption checks admin option", async () => { - const db = createEngineDB(sql, schema); - const role = await db.createUser({ - name: "admin-role", - }); - const admin = await db.createUser({ name: "admin-member" }); - - await db.addRoleMember(role.id, admin.id, true); - - const hasAdmin = await db.hasAdminOption(admin.id, role.id); - expect(hasAdmin).toBe(true); - }); -}); - -// --------------------------------------------------------------------------- -// Memory Tests -// --------------------------------------------------------------------------- -describe("memory ops", () => { - let testPrincipalId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - // Create a superuser for memory tests (bypasses RLS) - const superuser = await db.createSuperuser("memory-test-admin"); - testPrincipalId = superuser.id; - }); - - test("createMemory creates a memory", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const memory = await db.createMemory({ - content: "Test memory content", - meta: { key: "value" }, - tree: "test.memories", - }); - - expect(memory.id).toBeDefined(); - expect(memory.content).toBe("Test memory content"); - expect(memory.meta).toEqual({ key: "value" }); - expect(memory.tree).toBe("test.memories"); - expect(memory.hasEmbedding).toBe(false); - expect(memory.createdAt).toBeInstanceOf(Date); - }); - - test("createMemory with temporal point-in-time", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const now = new Date(); - const memory = await db.createMemory({ - content: "Point in time memory", - temporal: { start: now }, - }); - - expect(memory.temporal).not.toBeNull(); - expect(memory.temporal!.start.getTime()).toBe( - memory.temporal!.end.getTime(), - ); - }); - - test("createMemory with temporal range", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const start = new Date("2024-01-01"); - const end = new Date("2024-01-02"); - const memory = await db.createMemory({ - content: "Range memory", - temporal: { start, end }, - }); - - expect(memory.temporal).not.toBeNull(); - expect(memory.temporal!.start.getTime()).toBe(start.getTime()); - }); - - test("getMemory returns memory by ID", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ content: "Get test" }); - const fetched = await db.getMemory(created.id); - - expect(fetched).not.toBeNull(); - expect(fetched!.id).toBe(created.id); - expect(fetched!.content).toBe("Get test"); - }); - - test("updateMemory updates content", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ content: "Original" }); - const updated = await db.updateMemory(created.id, { content: "Updated" }); - - expect(updated).not.toBeNull(); - expect(updated!.content).toBe("Updated"); - }); - - test("updateMemory updates meta", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ - content: "Meta test", - meta: { old: true }, - }); - const updated = await db.updateMemory(created.id, { meta: { new: true } }); - - expect(updated!.meta).toEqual({ new: true }); - }); - - test("deleteMemory removes memory", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ content: "Delete test" }); - const result = await db.deleteMemory(created.id); - - expect(result).toBe(true); - - const fetched = await db.getMemory(created.id); - expect(fetched).toBeNull(); - }); - - test("deleteTree removes memories under path", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ content: "Tree 1", tree: "delete.tree.a" }); - await db.createMemory({ content: "Tree 2", tree: "delete.tree.b" }); - await db.createMemory({ content: "Other", tree: "other.tree" }); - - const result = await db.deleteTree("delete.tree"); - - expect(result.count).toBe(2); - }); - - test("moveTree moves memories to new path", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const m1 = await db.createMemory({ - content: "Move 1", - tree: "move.source", - }); - const m2 = await db.createMemory({ - content: "Move 2", - tree: "move.source.child", - }); - - const result = await db.moveTree("move.source", "move.destination"); - - expect(result.count).toBe(2); - - const fetched1 = await db.getMemory(m1.id); - expect(fetched1!.tree).toBe("move.destination"); - - const fetched2 = await db.getMemory(m2.id); - expect(fetched2!.tree).toBe("move.destination.child"); - }); - - test("moveTree dry-run counts without moving", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const m1 = await db.createMemory({ - content: "DryMove 1", - tree: "drymove.source", - }); - const m2 = await db.createMemory({ - content: "DryMove 2", - tree: "drymove.source.child", - }); - - // Dry-run preview uses countTree (same as RPC handler) - const preview = await db.countTree("drymove.source"); - expect(preview.count).toBe(2); - - // Verify memories were NOT moved - const fetched1 = await db.getMemory(m1.id); - expect(fetched1!.tree).toBe("drymove.source"); - - const fetched2 = await db.getMemory(m2.id); - expect(fetched2!.tree).toBe("drymove.source.child"); - }); - - test("countTree returns accurate count above 1000 (TNT-59 regression)", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Insert 1500 memories under a tree path. The bug capped the dry-run - // count at 1000 because the handler used searchMemories with limit:1000. - const total = 1500; - const batchSize = 500; - for (let start = 0; start < total; start += batchSize) { - const batch = Array.from({ length: batchSize }, (_, i) => ({ - content: `Bulk ${start + i}`, - tree: "bulk.count.regression", - })); - await db.batchCreateMemories(batch); - } - - // Sanity: searchMemories with limit:1000 (the old, buggy preview) caps at 1000. - const cappedPreview = await db.searchMemories({ - tree: "bulk.count.regression", - limit: 1000, - }); - expect(cappedPreview.total).toBe(1000); - - // countTree returns the true count, unbounded. - const count = await db.countTree("bulk.count.regression"); - expect(count.count).toBe(total); - - // Cleanup so subsequent tree-related tests aren't affected. - const deleted = await db.deleteTree("bulk.count.regression"); - expect(deleted.count).toBe(total); - }); - - test("countTree includes descendants and is empty for unknown paths", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ content: "A", tree: "count.tree" }); - await db.createMemory({ content: "B", tree: "count.tree.child" }); - await db.createMemory({ content: "C", tree: "count.tree.child.deep" }); - await db.createMemory({ content: "D", tree: "count.other" }); - - const inside = await db.countTree("count.tree"); - expect(inside.count).toBe(3); - - const empty = await db.countTree("count.does.not.exist"); - expect(empty.count).toBe(0); - }); - - test("batchCreateMemories creates multiple memories", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const ids = await db.batchCreateMemories([ - { content: "Batch 1", tree: "batch" }, - { content: "Batch 2", tree: "batch" }, - { content: "Batch 3", tree: "batch" }, - ]); - - expect(ids).toHaveLength(3); - - for (const id of ids) { - const memory = await db.getMemory(id); - expect(memory).not.toBeNull(); - } - }); - - test("batchCreateMemories skips duplicate ids without rolling back", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Two distinct UUIDv7s plus a duplicate of the first. The duplicate - // would otherwise hit the memory_pkey unique constraint and abort the - // transaction, taking the unique siblings down with it. - const idA = "019ddff0-0000-7000-8000-000000000a01"; - const idB = "019ddff0-0000-7000-8000-000000000b01"; - - const ids = await db.batchCreateMemories([ - { id: idA, content: "Dup batch A", tree: "dup.batch" }, - { id: idA, content: "Dup batch A redux", tree: "dup.batch" }, - { id: idB, content: "Dup batch B", tree: "dup.batch" }, - ]); - - // Only the unique inserts are returned — the duplicate is silently dropped. - expect(ids).toEqual([idA, idB]); - - // Both unique rows landed; the first content wins (dup is skipped, not updated). - const a = await db.getMemory(idA); - expect(a?.content).toBe("Dup batch A"); - const b = await db.getMemory(idB); - expect(b?.content).toBe("Dup batch B"); - }); - - test("batchCreateMemories tolerates ids that already exist in the table", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const id = "019ddff0-0000-7000-8000-000000000c01"; - await db.batchCreateMemories([ - { id, content: "Existing row", tree: "dup.preexisting" }, - ]); - - // Re-submitting a batch that includes the existing id should succeed, - // returning only the genuinely new ids and leaving the existing row's - // content untouched. - const newId = "019ddff0-0000-7000-8000-000000000c02"; - const ids = await db.batchCreateMemories([ - { id, content: "Re-attempted insert", tree: "dup.preexisting" }, - { - id: newId, - content: "Sibling that should land", - tree: "dup.preexisting", - }, - ]); - - expect(ids).toEqual([newId]); - const original = await db.getMemory(id); - expect(original?.content).toBe("Existing row"); - }); - - test("getTree returns tree structure", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ content: "Tree test 1", tree: "gettree.a.b" }); - await db.createMemory({ content: "Tree test 2", tree: "gettree.a.c" }); - - const tree = await db.getTree({ tree: "gettree" }); - - expect(tree.length).toBeGreaterThan(0); - expect(tree.some((n) => n.path === "gettree.a")).toBe(true); - }); - - test("searchMemories with filter-only returns results", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Create test memories - await db.createMemory({ - content: "Filter search test 1", - tree: "search.filter", - meta: { type: "test" }, - }); - await db.createMemory({ - content: "Filter search test 2", - tree: "search.filter", - meta: { type: "test" }, - }); - - const result = await db.searchMemories({ - tree: "search.filter", - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(2); - expect(result.results[0]!.score).toBe(1.0); // Filter-only uses score 1.0 - }); - - test("searchMemories with meta filter", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "Meta filter test", - tree: "search.meta", - meta: { category: "important", priority: 1 }, - }); - await db.createMemory({ - content: "Meta filter other", - tree: "search.meta", - meta: { category: "other" }, - }); - - const result = await db.searchMemories({ - meta: { category: "important" }, - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(1); - expect(result.results.some((r) => r.content === "Meta filter test")).toBe( - true, - ); - }); - - test("searchMemories with fulltext (BM25)", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "PostgreSQL is a powerful relational database", - tree: "search.bm25", - }); - await db.createMemory({ - content: "Redis is an in-memory key-value store", - tree: "search.bm25", - }); - - const result = await db.searchMemories({ - fulltext: "PostgreSQL database", - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(1); - expect(result.results[0]!.content).toContain("PostgreSQL"); - expect(result.results[0]!.score).toBeGreaterThan(0); - }); - - test("searchMemories with tree pattern (lquery)", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "Lquery test a.b", - tree: "lquery.a.b", - }); - await db.createMemory({ - content: "Lquery test a.c", - tree: "lquery.a.c", - }); - await db.createMemory({ - content: "Lquery test other", - tree: "other.path", - }); - - const result = await db.searchMemories({ - tree: "lquery.*", - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(2); - expect(result.results.every((r) => r.tree.startsWith("lquery"))).toBe(true); - }); - - test("searchMemories with temporal contains filter", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const jan1 = new Date("2024-01-01"); - const jan15 = new Date("2024-01-15"); - const feb1 = new Date("2024-02-01"); - - await db.createMemory({ - content: "January event", - tree: "search.temporal", - temporal: { start: jan1, end: feb1 }, - }); - await db.createMemory({ - content: "Point in time event", - tree: "search.temporal", - temporal: { start: jan15 }, - }); - - // Search for events containing Jan 10 - const result = await db.searchMemories({ - temporal: { contains: new Date("2024-01-10") }, - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(1); - expect(result.results.some((r) => r.content === "January event")).toBe( - true, - ); - }); - - test("searchMemories orderBy asc/desc", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Create memories with slight delay to ensure different timestamps - const m1 = await db.createMemory({ - content: "Order test first", - tree: "search.order", - }); - const m2 = await db.createMemory({ - content: "Order test second", - tree: "search.order", - }); - - // Descending (default) - newest first - const descResult = await db.searchMemories({ - tree: "search.order", - orderBy: "desc", - limit: 10, - }); - expect(descResult.results[0]!.id).toBe(m2.id); - - // Ascending - oldest first - const ascResult = await db.searchMemories({ - tree: "search.order", - orderBy: "asc", - limit: 10, - }); - expect(ascResult.results[0]!.id).toBe(m1.id); - }); - - test("searchMemories with grep filter", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "Error code ERR-42 occurred in production", - tree: "search.grep", - }); - await db.createMemory({ - content: "Warning code WARN-7 in staging", - tree: "search.grep", - }); - await db.createMemory({ - content: "All systems operational", - tree: "search.grep", - }); - - // Regex matching "ERR-\d+" should only return the first memory - const result = await db.searchMemories({ - grep: "ERR-\\d+", - tree: "search.grep", - limit: 10, - }); - - expect(result.results.length).toBe(1); - expect(result.results[0]!.content).toContain("ERR-42"); - }); - - test("searchMemories with grep + fulltext", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "PostgreSQL version 18.1 released with new features", - tree: "search.grepfull", - }); - await db.createMemory({ - content: "PostgreSQL conference announced for next year", - tree: "search.grepfull", - }); - - // BM25 matches both on "PostgreSQL", but grep narrows to version pattern - const result = await db.searchMemories({ - fulltext: "PostgreSQL", - grep: "version \\d+\\.\\d+", - limit: 10, - }); - - expect(result.results.length).toBe(1); - expect(result.results[0]!.content).toContain("version 18.1"); - }); - - test("searchMemories grep is case-insensitive", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "TypeScript is great", - tree: "search.grepcase", - }); - await db.createMemory({ - content: "typescript lowercase", - tree: "search.grepcase", - }); - - // Case-insensitive: matches both "TypeScript" and "typescript" - const result = await db.searchMemories({ - grep: "TypeScript", - tree: "search.grepcase", - limit: 10, - }); - - expect(result.results.length).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// Transaction Tests -// --------------------------------------------------------------------------- -describe("withTransaction", () => { - test("executes multiple ops atomically", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("tx-test-admin"); - db.setUser(superuser.id); - - const result = await db.withTransaction("write", async (txDb) => { - const m1 = await txDb.createMemory({ - content: "TX Memory 1", - tree: "tx", - }); - const m2 = await txDb.createMemory({ - content: "TX Memory 2", - tree: "tx", - }); - return [m1.id, m2.id]; - }); - - expect(result).toHaveLength(2); - - // Verify both were created - for (const id of result) { - const memory = await db.getMemory(id); - expect(memory).not.toBeNull(); - } - }); - - test("rolls back on error", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("rollback-test-admin"); - db.setUser(superuser.id); - - let createdId: string | null = null; - - try { - await db.withTransaction("write", async (txDb) => { - const m = await txDb.createMemory({ - content: "Rollback test", - tree: "rollback", - }); - createdId = m.id; - throw new Error("Intentional error"); - }); - } catch { - // Expected - } - - // Memory should not exist (rolled back) - if (createdId) { - const memory = await db.getMemory(createdId); - expect(memory).toBeNull(); - } - }); -}); diff --git a/packages/engine/db.ts b/packages/engine/db.ts deleted file mode 100644 index 816fe8b..0000000 --- a/packages/engine/db.ts +++ /dev/null @@ -1,183 +0,0 @@ -import type { SQL } from "bun"; -import { deriveContext, setLocalEngineTimeouts } from "./ops/_tx"; -import { type ApiKeyOps, apiKeyOps } from "./ops/api-key"; -import { type GrantOps, grantOps } from "./ops/grant"; -import { type MemoryOps, memoryOps } from "./ops/memory"; -import { type OwnerOps, ownerOps } from "./ops/owner"; -import { type RoleOps, roleOps } from "./ops/role"; -import { type UserOps, userOps } from "./ops/user"; -import type { OpsContext } from "./types"; - -export interface CreateEngineDBOptions { - /** Shard number for pgDog routing (future use) */ - shard?: number; -} - -/** - * All ops combined - */ -type AllOps = UserOps & ApiKeyOps & GrantOps & OwnerOps & RoleOps & MemoryOps; - -/** - * EngineDB interface - explicit type to avoid circular reference issues - */ -export interface EngineDB extends AllOps { - setUser(id: string): void; - getUserId(): string | null; - getSchema(): string; - getEngineSlug(): string; - withTransaction( - mode: "read" | "write", - fn: (db: EngineDB) => Promise, - ): Promise; -} - -/** - * Compose all ops into a single object - */ -function composeOps(ctx: OpsContext, engineSlug: string): AllOps { - return { - ...userOps(ctx), - ...apiKeyOps(ctx, engineSlug), - ...grantOps(ctx), - ...ownerOps(ctx), - ...roleOps(ctx), - ...memoryOps(ctx), - }; -} - -/** - * Extract engine slug from schema name (e.g., "me_abc123xyz789" -> "abc123xyz789") - */ -function extractSlugFromSchema(schema: string): string { - if (schema.startsWith("me_")) { - return schema.slice(3); - } - throw new Error(`Invalid schema name: ${schema} (must start with "me_")`); -} - -/** - * Create an EngineDB instance for a specific engine schema. - * - * EngineDB is the database abstraction layer for a single memory engine. - * It encapsulates all database operations and handles transaction management, - * role-based access control, and RLS context setup. - * - * @param sql - Database connection pool - * @param schema - Engine schema name (e.g., "me_abc123xyz789") - * @param options - Optional configuration (shard number for future pgDog routing) - */ -export function createEngineDB( - sql: SQL, - schema: string, - options?: CreateEngineDBOptions, -): EngineDB { - let userId: string | null = null; - const engineSlug = extractSlugFromSchema(schema); - - const ctx: OpsContext = { - sql, - schema, - shard: options?.shard, - inTransaction: false, - getUserId: () => userId, - }; - - const ops = composeOps(ctx, engineSlug); - - const db: EngineDB = { - ...ops, - - /** - * Set the current user ID for RLS context. - * This should be called after authentication, before making database calls. - */ - setUser(id: string): void { - userId = id; - }, - - /** - * Get the current user ID - */ - getUserId(): string | null { - return userId; - }, - - /** - * Get the schema name for this engine - */ - getSchema(): string { - return schema; - }, - - /** - * Get the engine slug (for API key generation) - */ - getEngineSlug(): string { - return engineSlug; - }, - - /** - * Execute multiple operations within a single transaction. - * - * Use this for batch operations that need to be atomic. - * Each operation inside the transaction will use the appropriate role - * (me_ro for reads, me_rw for writes). - * - * @param mode - "read" for read-only transaction, "write" for read-write - * @param fn - Function receiving a transactional EngineDB instance - */ - async withTransaction( - mode: "read" | "write", - fn: (db: EngineDB) => Promise, - ): Promise { - const role = mode === "read" ? "me_ro" : "me_rw"; - - return sql.begin(async (tx) => { - // Set up transaction context - if (ctx.shard !== undefined) { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${ctx.shard}`); - } - await setLocalEngineTimeouts(tx); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe(`SET LOCAL ROLE ${role}`); - if (userId) { - await tx`SELECT set_config('me.user_id', ${userId}, true)`; - } - - // Create a derived context for the transaction - const txCtx = deriveContext(ctx, tx); - const txOps = composeOps(txCtx, engineSlug); - - // Create a transactional EngineDB instance - const txDb: EngineDB = { - ...txOps, - setUser(id: string): void { - userId = id; - }, - getUserId(): string | null { - return userId; - }, - getSchema(): string { - return schema; - }, - getEngineSlug(): string { - return engineSlug; - }, - // Nested withTransaction just runs the function directly - // (already in a transaction) - async withTransaction( - _mode: "read" | "write", - nestedFn: (db: EngineDB) => Promise, - ): Promise { - return nestedFn(txDb); - }, - }; - - return fn(txDb); - }); - }, - }; - - return db; -} diff --git a/packages/engine/index.ts b/packages/engine/index.ts index f2ca310..a693619 100644 --- a/packages/engine/index.ts +++ b/packages/engine/index.ts @@ -1,46 +1,8 @@ -// Main exports - -// New core control-plane + space data-plane layers (target the core / me_ -// schemas via SQL functions). Namespaced to avoid clashing with the legacy flat -// exports below during the migration: core.coreStore, space.spaceStore, etc. +// The engine package is the runtime layer over the new-model schemas: +// - core: control plane (core schema) — spaces, principals, membership, +// groups, tree-access grants, api keys. +// - space: data plane (per-space me_ schema) — memory CRUD, tree, search. +// Namespaced so callers pick a plane explicitly: `core.coreStore`, `space.spaceStore`. +// Subpath imports (`@memory.build/engine/core`, `/space`) are equivalent. export * as core from "./core"; -export { - type CreateEngineDBOptions, - createEngineDB, - type EngineDB, -} from "./db"; -// Re-export migrate module -export * from "./migrate"; export * as space from "./space"; -// Type exports -export { - type ApiKey, - type CreateApiKeyParams, - type CreateApiKeyResult, - type CreateMemoryParams, - type CreateUserParams, - type GetTreeParams, - type GrantTreeAccessParams, - type Memory, - NotImplementedError, - type OpsContext, - type RoleInfo, - type RoleMember, - type SearchParams, - type SearchResult, - type SearchResultItem, - type SearchWeights, - type TemporalFilter, - type TreeGrant, - type TreeNode, - type TreeOwner, - type UpdateMemoryParams, - type User, - type ValidateApiKeyResult, -} from "./types"; -// Utility exports -export { - extractEngineSlug, - formatApiKey, - parseApiKey, -} from "./util"; diff --git a/packages/engine/migrate/bootstrap.ts b/packages/engine/migrate/bootstrap.ts deleted file mode 100644 index e3b9805..0000000 --- a/packages/engine/migrate/bootstrap.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { SQL, semver } from "bun"; -import { setLocalEngineTimeouts } from "../ops/_tx"; - -export async function bootstrap(sql: SQL): Promise { - await sql.begin(async (tx) => { - await setLocalEngineTimeouts(tx); - await ensurePrerequisites(tx); - await ensureRoles(tx); - }); -} - -async function ensurePrerequisites(sql: SQL): Promise { - const [{ server_version_num }] = await sql` - select current_setting('server_version_num')::int as server_version_num - `; - if (server_version_num < 180000) { - throw new Error( - `PostgreSQL version 18 or higher is required (found ${server_version_num})`, - ); - } - - await ensureExtension(sql, "citext", "1.6"); - await ensureExtension(sql, "ltree", "1.3"); - await ensureExtension(sql, "vector", "0.8.2"); - await ensureExtension(sql, "pg_textsearch", "1.1.0"); -} - -async function ensureExtension( - sql: SQL, - name: string, - minVersion: string, -): Promise { - const [installed] = await sql` - select extversion from pg_extension where extname = ${name} - `; - - if (installed) { - if (semver.order(installed.extversion, minVersion) >= 0) { - return; - } - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required (found ${installed.extversion} installed)`, - ); - } - - const [available] = await sql` - select default_version - from pg_available_extensions - where name = ${name} - `; - - if (!available || semver.order(available.default_version, minVersion) < 0) { - const found = available - ? `found ${available.default_version} available` - : "not available"; - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required (${found})`, - ); - } - - try { - await sql`create extension if not exists ${sql(name)}`; - } catch (error: unknown) { - // Ignore duplicate extension errors (race condition in concurrent calls) - if ( - error instanceof SQL.PostgresError && - error.errno === "23505" && - error.constraint === "pg_extension_name_index" - ) { - return; - } - throw error; - } -} - -async function ensureRoles(sql: SQL): Promise { - await sql.unsafe(` - do $block$ - declare - _roles text[] = array['me_ro', 'me_rw', 'me_embed']; - _role text; - _sql text; - begin - for _role in select * from unnest(_roles) loop - perform - from pg_roles r - where r.rolname = _role; - if found then - continue; - end if; - _sql = format($sql$create role %I nologin$sql$, _role); - execute _sql; - _sql = format($sql$grant %I to %I$sql$, _role, current_user); - execute _sql; - end loop; - end; - $block$; - `); -} diff --git a/packages/engine/migrate/discover.test.ts b/packages/engine/migrate/discover.test.ts deleted file mode 100644 index 8f9233c..0000000 --- a/packages/engine/migrate/discover.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { - isValidEngineSchema, - isValidSlug, - schemaToSlug, - slugToSchema, -} from "./discover"; - -describe("isValidEngineSchema", () => { - test("valid 12-char lowercase alphanumeric", () => { - expect(isValidEngineSchema("me_abcdef123456")).toBe(true); - }); - - test("valid all digits", () => { - expect(isValidEngineSchema("me_000000000000")).toBe(true); - }); - - test("valid all letters", () => { - expect(isValidEngineSchema("me_abcdefghijkl")).toBe(true); - }); - - test("rejects too short", () => { - expect(isValidEngineSchema("me_abc")).toBe(false); - }); - - test("rejects too long", () => { - expect(isValidEngineSchema("me_abcdef1234567")).toBe(false); - }); - - test("rejects uppercase", () => { - expect(isValidEngineSchema("me_ABCDEF123456")).toBe(false); - }); - - test("rejects wrong prefix", () => { - expect(isValidEngineSchema("xx_abcdef123456")).toBe(false); - }); - - test("rejects no prefix", () => { - expect(isValidEngineSchema("abcdef123456")).toBe(false); - }); - - test("rejects empty string", () => { - expect(isValidEngineSchema("")).toBe(false); - }); - - test("rejects special characters", () => { - expect(isValidEngineSchema("me_abcdef12345!")).toBe(false); - }); - - test("rejects public schema", () => { - expect(isValidEngineSchema("public")).toBe(false); - }); - - test("rejects embedding schema", () => { - expect(isValidEngineSchema("embedding")).toBe(false); - }); -}); - -describe("isValidSlug", () => { - test("valid 12-char lowercase alphanumeric", () => { - expect(isValidSlug("abcdef123456")).toBe(true); - }); - - test("valid all digits", () => { - expect(isValidSlug("000000000000")).toBe(true); - }); - - test("rejects too short", () => { - expect(isValidSlug("abc")).toBe(false); - }); - - test("rejects too long", () => { - expect(isValidSlug("abcdef1234567")).toBe(false); - }); - - test("rejects uppercase", () => { - expect(isValidSlug("ABCDEF123456")).toBe(false); - }); - - test("rejects special characters", () => { - expect(isValidSlug("abcdef12345!")).toBe(false); - }); - - test("rejects empty string", () => { - expect(isValidSlug("")).toBe(false); - }); - - test("rejects hyphens", () => { - expect(isValidSlug("abc-def-12345")).toBe(false); - }); -}); - -describe("slugToSchema / schemaToSlug", () => { - test("round-trip slug → schema → slug", () => { - const slug = "abcdef123456"; - expect(schemaToSlug(slugToSchema(slug))).toBe(slug); - }); - - test("slugToSchema adds me_ prefix", () => { - expect(slugToSchema("abcdef123456")).toBe("me_abcdef123456"); - }); - - test("schemaToSlug removes me_ prefix", () => { - expect(schemaToSlug("me_abcdef123456")).toBe("abcdef123456"); - }); -}); diff --git a/packages/engine/migrate/discover.ts b/packages/engine/migrate/discover.ts deleted file mode 100644 index 27f0adf..0000000 --- a/packages/engine/migrate/discover.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { SQL } from "bun"; - -const ENGINE_SCHEMA_RE = /^me_[a-z0-9]{12}$/; -const SLUG_RE = /^[a-z0-9]{12}$/; - -export async function discoverEngineSchemas(sql: SQL): Promise { - const rows = await sql` - select nspname - from pg_namespace - where nspname ~ '^me_[a-z0-9]{12}$' - order by nspname - `; - return rows.map((r: { nspname: string }) => r.nspname); -} - -export function isValidEngineSchema(name: string): boolean { - return ENGINE_SCHEMA_RE.test(name); -} - -export function isValidSlug(slug: string): boolean { - return SLUG_RE.test(slug); -} - -export function slugToSchema(slug: string): string { - return `me_${slug}`; -} - -export function schemaToSlug(schema: string): string { - return schema.slice(3); -} - -export async function assertEngineSchema( - sql: SQL, - schema: string, -): Promise { - if (!isValidEngineSchema(schema)) { - throw new Error( - `Invalid engine schema: "${schema}" — must match me_[a-z0-9]{12}`, - ); - } - - const [row] = await sql` - select 1 from pg_namespace where nspname = ${schema} - `; - if (!row) { - throw new Error(`Engine schema "${schema}" does not exist`); - } -} diff --git a/packages/engine/migrate/index.ts b/packages/engine/migrate/index.ts deleted file mode 100644 index 758524e..0000000 --- a/packages/engine/migrate/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -export { bootstrap } from "./bootstrap"; -export { - assertEngineSchema, - discoverEngineSchemas, - isValidEngineSchema, - isValidSlug, - schemaToSlug, - slugToSchema, -} from "./discover"; -export type { ProvisionResult } from "./provision"; -export { provisionEngine } from "./provision"; -export type { MigrateResult } from "./runner"; -export { - dryRun, - getMigrations, - getVersion, - migrateAll, - migrateEngine, -} from "./runner"; -export type { EngineConfig, ResolvedConfig } from "./template"; -export { defaultConfig, resolveConfig, template } from "./template"; diff --git a/packages/engine/migrate/migrate.integration.test.ts b/packages/engine/migrate/migrate.integration.test.ts deleted file mode 100644 index feae8b0..0000000 --- a/packages/engine/migrate/migrate.integration.test.ts +++ /dev/null @@ -1,823 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { SQL } from "bun"; -import { bootstrap } from "./bootstrap"; -import { discoverEngineSchemas } from "./discover"; -import { provisionEngine } from "./provision"; -import { dryRun, getVersion, migrateAll, migrateEngine } from "./runner"; -import { - countMigrations, - getFunctions, - getIndexes, - getRoles, - getTableColumns, - schemaExists, - TestDatabase, - tableExists, -} from "./test-utils"; - -const testDb = new TestDatabase(); -let connectionString: string; -let sql: SQL; - -beforeAll(async () => { - connectionString = await testDb.create(); - sql = new SQL(connectionString); - await bootstrap(sql); -}); - -afterAll(async () => { - await sql.close(); - await testDb.drop(); -}); - -// --------------------------------------------------------------------------- -// Bootstrap Tests -// --------------------------------------------------------------------------- -describe("bootstrap", () => { - test("creates extensions", async () => { - const rows = await sql` - select extname from pg_extension - where extname in ('citext', 'ltree', 'vector', 'pg_textsearch') - order by extname - `; - const names = rows.map((r: { extname: string }) => r.extname); - expect(names).toEqual(["citext", "ltree", "pg_textsearch", "vector"]); - }); - - test("creates roles", async () => { - const roles = await getRoles(sql, "me_ro", "me_rw", "me_embed"); - expect(roles).toHaveLength(3); - for (const role of roles) { - expect(role.rolcanlogin).toBe(false); - } - }); - - test("does not create embedding schema", async () => { - expect(await schemaExists(sql, "embedding")).toBe(false); - }); - - test("is idempotent", async () => { - // Run bootstrap again — should not error - await bootstrap(sql); - - const rows = await sql` - select extname from pg_extension - where extname in ('citext', 'ltree', 'vector', 'pg_textsearch') - `; - expect(rows).toHaveLength(4); - }); -}); - -// --------------------------------------------------------------------------- -// Single-Engine Migration Tests -// --------------------------------------------------------------------------- -describe("single-engine migration", () => { - const slug = "testengine01"; - const schema = `me_${slug}`; - - beforeAll(async () => { - await provisionEngine(sql, slug, undefined, "0.1.0"); - }); - - test("creates all tables", async () => { - for (const table of [ - "memory", - "user", - "api_key", - "tree_grant", - "role_membership", - "tree_owner", - "migration", - "embedding_queue", - ]) { - expect(await tableExists(sql, schema, table)).toBe(true); - } - }); - - test("creates memory indexes", async () => { - const indexes = await getIndexes(sql, schema, "memory"); - expect(indexes).toContain("memory_meta_gin_idx"); - expect(indexes).toContain("memory_temporal_gist_idx"); - expect(indexes).toContain("memory_content_bm25_idx"); - expect(indexes).toContain("memory_embedding_hnsw_idx"); - expect(indexes).toContain("memory_tree_gist_idx"); - expect(indexes).toContain("memory_null_embedding_idx"); - }); - - test("is idempotent", async () => { - const result = await migrateEngine(sql, schema, undefined, "0.1.0"); - expect(result.status).toBe("ok"); - expect(result.applied).toHaveLength(0); - expect(await countMigrations(sql, schema)).toBe(7); - }); - - test("records migration metadata", async () => { - const rows = await sql.unsafe(` - select name, applied_at_version, applied_at - from ${schema}.migration - order by name - `); - expect(rows).toHaveLength(7); - for (const row of rows) { - expect(row.applied_at_version).toBe("0.1.0"); - expect(row.applied_at).toBeTruthy(); - } - }); - - test("template substitution with custom config", async () => { - // Verify the memory table was created (uses embedding_dimensions template var) - const cols = await getTableColumns(sql, schema, "memory"); - const embCol = cols.find((c) => c.column_name === "embedding"); - expect(embCol).toBeTruthy(); - }); - - test("memory trigger nulls embedding on content change", async () => { - // Insert a memory with a fake embedding - const dims = 1536; - const embedding = `[${Array(dims).fill(0.1).join(",")}]`; - await sql.unsafe(` - insert into ${schema}.memory (content, embedding) - values ('original content', '${embedding}') - `); - - const [before] = await sql.unsafe(` - select id, embedding from ${schema}.memory - where content = 'original content' - `); - expect(before.embedding).not.toBeNull(); - - // Update content without explicitly setting embedding - await sql.unsafe(` - update ${schema}.memory - set content = 'updated content' - where id = '${before.id}' - `); - - const [after] = await sql.unsafe(` - select embedding from ${schema}.memory - where id = '${before.id}' - `); - expect(after.embedding).toBeNull(); - }); - - test("memory trigger increments embedding_version", async () => { - await sql.unsafe(` - insert into ${schema}.memory (content) - values ('version test content') - `); - - const [before] = await sql.unsafe(` - select id, embedding_version from ${schema}.memory - where content = 'version test content' - `); - expect(before.embedding_version).toBe(1); - - await sql.unsafe(` - update ${schema}.memory - set content = 'version test updated' - where id = '${before.id}' - `); - - const [after] = await sql.unsafe(` - select embedding_version from ${schema}.memory - where id = '${before.id}' - `); - expect(after.embedding_version).toBe(2); - }); - - test("memory trigger preserves embedding when explicitly set", async () => { - const dims = 1536; - const embedding = `[${Array(dims).fill(0.2).join(",")}]`; - await sql.unsafe(` - insert into ${schema}.memory (content, embedding) - values ('preserve test', '${embedding}') - `); - - const [{ id }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'preserve test' - `); - - const newEmbedding = `[${Array(dims).fill(0.3).join(",")}]`; - await sql.unsafe(` - update ${schema}.memory - set content = 'preserve test updated', embedding = '${newEmbedding}' - where id = '${id}' - `); - - const [after] = await sql.unsafe(` - select embedding from ${schema}.memory where id = '${id}' - `); - expect(after.embedding).not.toBeNull(); - }); - - test("auth tables have correct structure", async () => { - // User table: thing that accesses memories (or a role if can_login = false) - const userCols = await getTableColumns(sql, schema, "user"); - const colNames = userCols.map((c) => c.column_name); - expect(colNames).toContain("id"); - expect(colNames).toContain("name"); - expect(colNames).toContain("identity_id"); - expect(colNames).toContain("can_login"); - expect(colNames).toContain("superuser"); - expect(colNames).toContain("createrole"); - expect(colNames).toContain("created_at"); - expect(colNames).toContain("updated_at"); - - // api_key table exists in engine (user-scoped, engine-scoped) - expect(await tableExists(sql, schema, "api_key")).toBe(true); - const apiKeyCols = await getTableColumns(sql, schema, "api_key"); - const apiKeyColNames = apiKeyCols.map((c) => c.column_name); - expect(apiKeyColNames).toContain("user_id"); - expect(apiKeyColNames).toContain("lookup_id"); - expect(apiKeyColNames).toContain("key_hash"); - }); - - test("RLS policies enabled on memory", async () => { - const [row] = await sql` - select relrowsecurity - from pg_class c - join pg_namespace n on n.oid = c.relnamespace - where n.nspname = ${schema} and c.relname = 'memory' - `; - expect(row.relrowsecurity).toBe(true); - }); - - test("functions have explicit search_path", async () => { - const funcs = await getFunctions(sql, schema); - for (const func of funcs) { - expect(func.proconfig).toBeTruthy(); - const hasSearchPath = func.proconfig!.some((c: string) => - c.startsWith("search_path="), - ); - expect(hasSearchPath).toBe(true); - } - }); - - test("meta jsonb must be object", async () => { - expect(async () => { - await sql.unsafe(` - insert into ${schema}.memory (content, meta) values ('test', '[]') - `); - }).toThrow(); - }); - - test("temporal constraints enforced", async () => { - // Point-in-time: valid - await sql.unsafe(` - insert into ${schema}.memory (content, temporal) - values ('point', '[2024-01-01, 2024-01-01]') - `); - - // Range: valid - await sql.unsafe(` - insert into ${schema}.memory (content, temporal) - values ('range', '[2024-01-01, 2024-06-01)') - `); - - // Invalid: exclusive lower bound - expect(async () => { - await sql.unsafe(` - insert into ${schema}.memory (content, temporal) - values ('bad', '(2024-01-01, 2024-06-01)') - `); - }).toThrow(); - }); - - test("embedding_version defaults to 1", async () => { - await sql.unsafe(` - insert into ${schema}.memory (content) values ('ev default test') - `); - const [row] = await sql.unsafe(` - select embedding_version from ${schema}.memory - where content = 'ev default test' - `); - expect(row.embedding_version).toBe(1); - }); - - test("enqueue trigger fires on insert", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - await sql.unsafe(` - insert into ${schema}.memory (content) values ('trigger insert test') - `); - - const rows = await sql.unsafe(` - select memory_id, embedding_version - from ${schema}.embedding_queue - `); - expect(rows.length).toBeGreaterThanOrEqual(1); - }); - - test("enqueue trigger fires on content update", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - await sql.unsafe(` - insert into ${schema}.memory (content) values ('trigger update before') - `); - - // Clear queue entries from the insert - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - const [{ id }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'trigger update before' - `); - - await sql.unsafe(` - update ${schema}.memory set content = 'trigger update after' where id = '${id}' - `); - - const rows = await sql.unsafe(` - select embedding_version - from ${schema}.embedding_queue - `); - expect(rows.length).toBeGreaterThanOrEqual(1); - }); - - test("memory deletion cascades to embedding_queue", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - await sql.unsafe(` - insert into ${schema}.memory (content) values ('cascade test') - `); - - const [{ id }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'cascade test' - `); - - // Verify queue entry exists - const before = await sql.unsafe(` - select count(*)::int as cnt from ${schema}.embedding_queue where memory_id = '${id}' - `); - expect(before[0].cnt).toBeGreaterThanOrEqual(1); - - // Delete memory — queue entry should cascade - await sql.unsafe(`delete from ${schema}.memory where id = '${id}'`); - - const after = await sql.unsafe(` - select count(*)::int as cnt from ${schema}.embedding_queue where memory_id = '${id}' - `); - expect(after[0].cnt).toBe(0); - }); - - test("me_embed role has correct per-schema grants", async () => { - // Check schema usage - const [{ has_usage }] = await sql` - select has_schema_privilege('me_embed', ${schema}, 'USAGE') as has_usage - `; - expect(has_usage).toBe(true); - - // Check memory table privileges - const [{ has_select }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.memory`}, 'SELECT') as has_select - `; - expect(has_select).toBe(true); - - const [{ has_update }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.memory`}, 'UPDATE') as has_update - `; - expect(has_update).toBe(true); - - // Check embedding_queue table privileges - const [{ eq_select }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.embedding_queue`}, 'SELECT') as eq_select - `; - expect(eq_select).toBe(true); - - const [{ eq_update }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.embedding_queue`}, 'UPDATE') as eq_update - `; - expect(eq_update).toBe(true); - - const [{ eq_delete }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.embedding_queue`}, 'DELETE') as eq_delete - `; - expect(eq_delete).toBe(true); - - // Check claim function privilege - const [{ has_execute }] = await sql` - select has_function_privilege('me_embed', ${`${schema}.claim_embedding_batch(int, interval)`}, 'EXECUTE') as has_execute - `; - expect(has_execute).toBe(true); - }); - - test("me_rw cannot access embedding_queue", async () => { - const [{ has_select }] = await sql` - select has_table_privilege('me_rw', ${`${schema}.embedding_queue`}, 'SELECT') as has_select - `; - expect(has_select).toBe(false); - }); - - test("me_ro cannot access embedding_queue", async () => { - const [{ has_select }] = await sql` - select has_table_privilege('me_ro', ${`${schema}.embedding_queue`}, 'SELECT') as has_select - `; - expect(has_select).toBe(false); - }); - - test("embedding_queue has FK with ON DELETE CASCADE", async () => { - const fks = await sql` - select - tc.constraint_name, - rc.delete_rule - from information_schema.table_constraints tc - join information_schema.referential_constraints rc - on tc.constraint_name = rc.constraint_name - and tc.constraint_schema = rc.constraint_schema - where tc.table_schema = ${schema} - and tc.table_name = 'embedding_queue' - and tc.constraint_type = 'FOREIGN KEY' - `; - expect(fks).toHaveLength(1); - expect(fks[0].delete_rule).toBe("CASCADE"); - }); - - test("per-engine enqueue_embedding and claim_embedding_batch functions exist", async () => { - const funcs = await getFunctions(sql, schema); - const names = funcs.map((f) => f.proname); - expect(names).toContain("enqueue_embedding"); - expect(names).toContain("claim_embedding_batch"); - expect(names).toContain("prune_embedding_queue"); - }); - - test("me_embed can execute prune_embedding_queue", async () => { - const [{ has_execute }] = await sql` - select has_function_privilege( - 'me_embed', - ${`${schema}.prune_embedding_queue(interval)`}, - 'EXECUTE' - ) as has_execute - `; - expect(has_execute).toBe(true); - }); - - test("prune_embedding_queue removes terminal rows older than retention", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - // Insert a memory we can reference (avoid FK fan-out) - await sql.unsafe(` - insert into ${schema}.memory (content) values ('prune test memory') - `); - const [{ id: memId }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'prune test memory' - `); - - // Clear the auto-enqueued row from the trigger - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - // Old completed (should be pruned) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 1, 'completed', now() - interval '10 days') - `); - // Old failed (should be pruned) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 2, 'failed', now() - interval '10 days') - `); - // Old cancelled (should be pruned) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 3, 'cancelled', now() - interval '10 days') - `); - // Recent completed (should be kept) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 4, 'completed', now() - interval '1 day') - `); - // Old but outcome IS NULL — active queue item, must NOT be pruned - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 5, null, now() - interval '30 days') - `); - - const [{ pruned }] = await sql.unsafe( - `select ${schema}.prune_embedding_queue(interval '7 days') as pruned`, - ); - expect(Number(pruned)).toBe(3); - - const remaining = await sql.unsafe( - `select embedding_version, outcome from ${schema}.embedding_queue - where memory_id = '${memId}' - order by embedding_version`, - ); - expect(remaining).toHaveLength(2); - expect(remaining[0].embedding_version).toBe(4); - expect(remaining[0].outcome).toBe("completed"); - expect(remaining[1].embedding_version).toBe(5); - expect(remaining[1].outcome).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// Multi-Engine Tests -// --------------------------------------------------------------------------- -describe("multi-engine migration", () => { - const slugs = ["aaaa00000001", "aaaa00000002", "aaaa00000003"]; - const schemas = slugs.map((s) => `me_${s}`); - - beforeAll(async () => { - for (const slug of slugs) { - await provisionEngine(sql, slug, undefined, "0.1.0"); - } - }); - - test("migrateAll migrates multiple schemas", async () => { - // All already migrated, should be no-op - const results = await migrateAll(sql, schemas, undefined, "0.1.0"); - expect(results.size).toBe(3); - for (const [, result] of results) { - expect(result.status).toBe("ok"); - expect(result.applied).toHaveLength(0); - } - for (const schema of schemas) { - expect(await tableExists(sql, schema, "memory")).toBe(true); - } - }); - - test("migrateAll isolates failures", async () => { - const badSchema = "me_badschema000"; - // Don't create this schema — migration should fail - const allSchemas = [...schemas, badSchema]; - - const results = await migrateAll(sql, allSchemas, undefined, "0.1.0"); - - // Good schemas should succeed (already migrated, so 0 applied) - for (const schema of schemas) { - expect(results.get(schema)!.status).toBe("ok"); - } - - // Bad schema should error - expect(results.get(badSchema)!.status).toBe("error"); - expect(results.get(badSchema)!.error).toBeTruthy(); - }); - - test("concurrency control processes with concurrency=1", async () => { - const results = await migrateAll(sql, schemas, undefined, "0.1.0", { - concurrency: 1, - }); - expect(results.size).toBe(3); - for (const [, result] of results) { - expect(result.status).toBe("ok"); - } - }); -}); - -// --------------------------------------------------------------------------- -// Discovery Tests -// --------------------------------------------------------------------------- -describe("discovery", () => { - test("finds engine schemas", async () => { - // me_testengine01 was created earlier, plus multi schemas - const discovered = await discoverEngineSchemas(sql); - expect(discovered).toContain("me_testengine01"); - expect(discovered).toContain("me_aaaa00000001"); - }); - - test("ignores non-engine schemas", async () => { - const discovered = await discoverEngineSchemas(sql); - expect(discovered).not.toContain("public"); - // embedding schema no longer exists - expect(discovered).not.toContain("pg_catalog"); - }); - - test("returns sorted results", async () => { - const discovered = await discoverEngineSchemas(sql); - const sorted = [...discovered].sort(); - expect(discovered).toEqual(sorted); - }); -}); - -// --------------------------------------------------------------------------- -// Advisory Lock Tests -// --------------------------------------------------------------------------- -describe("advisory locks", () => { - test("concurrent migrateEngine on same schema — only one applies", async () => { - const slug = "locktest0001"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - - // Now run concurrent migrations (all should succeed with 0 applied due to idempotency) - const results = await Promise.all([ - migrateEngine(sql, schema, undefined, "0.1.0"), - migrateEngine(sql, schema, undefined, "0.1.0"), - migrateEngine(sql, schema, undefined, "0.1.0"), - ]); - - // All should complete (ok or skipped) - for (const result of results) { - expect(["ok", "skipped"]).toContain(result.status); - } - - // Exactly 7 migrations should exist - expect(await countMigrations(sql, schema)).toBe(7); - }); - - test("concurrent migrateEngine on different schemas — both proceed", async () => { - const slugA = "locktest0002"; - const slugB = "locktest0003"; - const schemaA = `me_${slugA}`; - const schemaB = `me_${slugB}`; - - // Provision both first - await Promise.all([ - provisionEngine(sql, slugA, undefined, "0.1.0"), - provisionEngine(sql, slugB, undefined, "0.1.0"), - ]); - - // Now run migrations (should be no-ops since provisioning ran them) - const [resultA, resultB] = await Promise.all([ - migrateEngine(sql, schemaA, undefined, "0.1.0"), - migrateEngine(sql, schemaB, undefined, "0.1.0"), - ]); - - expect(resultA.status).toBe("ok"); - expect(resultB.status).toBe("ok"); - }); -}); - -// --------------------------------------------------------------------------- -// Provisioning Tests -// --------------------------------------------------------------------------- -describe("provisioning", () => { - test("creates schema and runs all migrations", async () => { - const result = await provisionEngine( - sql, - "prov00000001", - undefined, - "0.1.0", - ); - expect(result.schema).toBe("me_prov00000001"); - expect(result.migrateResult.status).toBe("ok"); - expect(result.migrateResult.applied).toHaveLength(7); - expect(await schemaExists(sql, "me_prov00000001")).toBe(true); - expect(await tableExists(sql, "me_prov00000001", "memory")).toBe(true); - expect(await tableExists(sql, "me_prov00000001", "user")).toBe(true); - }); - - test("validates slug format", () => { - expect(provisionEngine(sql, "BAD", undefined, "0.1.0")).rejects.toThrow( - "Invalid engine slug", - ); - - expect( - provisionEngine(sql, "too-short", undefined, "0.1.0"), - ).rejects.toThrow("Invalid engine slug"); - }); - - test("fails if schema already exists", async () => { - await provisionEngine(sql, "prov00000002", undefined, "0.1.0"); - - await expect( - provisionEngine(sql, "prov00000002", undefined, "0.1.0"), - ).rejects.toThrow(); - }); - - test("creates version table", async () => { - const slug = "prov00000003"; - await provisionEngine(sql, slug, undefined, "0.1.0"); - expect(await tableExists(sql, `me_${slug}`, "version")).toBe(true); - expect(await getVersion(sql, `me_${slug}`)).toBe("0.1.0"); - }); -}); - -// --------------------------------------------------------------------------- -// Dry Run Tests -// --------------------------------------------------------------------------- -describe("dry run", () => { - test("shows all pending for new schema", async () => { - // Create a schema manually without running migrations (simulating a fresh schema) - const schema = "me_dryrun000001"; - await sql.unsafe(`create schema if not exists ${schema}`); - - const result = await dryRun(sql, schema); - expect(result.pending).toHaveLength(7); - expect(result.applied).toHaveLength(0); - }); - - test("shows none pending after full migration", async () => { - const slug = "dryrun000002"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - - const result = await dryRun(sql, schema); - expect(result.pending).toHaveLength(0); - expect(result.applied).toHaveLength(7); - }); -}); - -// --------------------------------------------------------------------------- -// Version Tracking Tests -// --------------------------------------------------------------------------- -describe("version tracking", () => { - test("applied_at_version records correctly", async () => { - const slug = "version00001"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "1.2.3"); - - const rows = await sql.unsafe(` - select applied_at_version from ${schema}.migration - `); - for (const row of rows) { - expect(row.applied_at_version).toBe("1.2.3"); - } - }); - - test("re-migrate with same migrations is no-op", async () => { - const slug = "version00002"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - - const result = await migrateEngine(sql, schema, undefined, "0.1.0"); - expect(result.applied).toHaveLength(0); - }); - - test("rejects downgrade", async () => { - const slug = "version00003"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.2.0"); - - await expect( - migrateEngine(sql, schema, undefined, "0.1.0"), - ).rejects.toThrow("older than database version"); - }); - - test("updates version on upgrade", async () => { - const slug = "version00004"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - expect(await getVersion(sql, schema)).toBe("0.1.0"); - - await migrateEngine(sql, schema, undefined, "0.2.0"); - expect(await getVersion(sql, schema)).toBe("0.2.0"); - }); - - test("getVersion returns current version", async () => { - const slug = "version00005"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "1.2.3"); - expect(await getVersion(sql, schema)).toBe("1.2.3"); - }); -}); - -// --------------------------------------------------------------------------- -// Cross-Engine Isolation Tests -// --------------------------------------------------------------------------- -describe("cross-engine isolation", () => { - const slugA = "isolate00001"; - const slugB = "isolate00002"; - const schemaA = `me_${slugA}`; - const schemaB = `me_${slugB}`; - - beforeAll(async () => { - await provisionEngine(sql, slugA, undefined, "0.1.0"); - await provisionEngine(sql, slugB, undefined, "0.1.0"); - }); - - test("data isolated between schemas", async () => { - await sql.unsafe(` - insert into ${schemaA}.memory (content) values ('only in A') - `); - - const rowsA = await sql.unsafe(` - select content from ${schemaA}.memory where content = 'only in A' - `); - expect(rowsA).toHaveLength(1); - - const rowsB = await sql.unsafe(` - select content from ${schemaB}.memory where content = 'only in A' - `); - expect(rowsB).toHaveLength(0); - }); - - test("embedding queue entries are per-engine", async () => { - await sql.unsafe(`delete from ${schemaA}.embedding_queue`); - await sql.unsafe(`delete from ${schemaB}.embedding_queue`); - - await sql.unsafe(` - insert into ${schemaA}.memory (content) values ('queue test A') - `); - await sql.unsafe(` - insert into ${schemaB}.memory (content) values ('queue test B') - `); - - const rowsA = await sql.unsafe(` - select memory_id from ${schemaA}.embedding_queue - `); - expect(rowsA).toHaveLength(1); - - const rowsB = await sql.unsafe(` - select memory_id from ${schemaB}.embedding_queue - `); - expect(rowsB).toHaveLength(1); - }); -}); diff --git a/packages/engine/migrate/migrations/001_updated_at.sql b/packages/engine/migrate/migrations/001_updated_at.sql deleted file mode 100644 index 4e11805..0000000 --- a/packages/engine/migrate/migrations/001_updated_at.sql +++ /dev/null @@ -1,10 +0,0 @@ --- generic trigger function to update updated_at timestamp -create function {{schema}}.update_updated_at() -returns trigger -as $func$ -begin - new.updated_at = pg_catalog.now(); - return new; -end; -$func$ language plpgsql volatile security definer -set search_path to {{schema}}, pg_temp; diff --git a/packages/engine/migrate/migrations/002_memory.sql b/packages/engine/migrate/migrations/002_memory.sql deleted file mode 100644 index 7c8a4de..0000000 --- a/packages/engine/migrate/migrations/002_memory.sql +++ /dev/null @@ -1,57 +0,0 @@ -create table {{schema}}.memory -( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) -, meta jsonb not null default '{}' -, tree ltree not null default ''::ltree -, temporal tstzrange -, content text not null -, embedding halfvec({{embedding_dimensions}}) -, embedding_version int4 not null default 1 -, embedding_attempts int4 not null default 0 -, embedding_last_error text -, created_at timestamptz not null default now() -, created_by uuid -, updated_at timestamptz -); - -grant select on {{schema}}.memory to me_ro; -grant select, insert, update, delete on {{schema}}.memory to me_rw; - --- index for faceted search -create index memory_meta_gin_idx on {{schema}}.memory using gin (meta); - --- index for temporal search -create index memory_temporal_gist_idx on {{schema}}.memory using gist (temporal) where (temporal is not null); - --- index for BM25 text search -create index memory_content_bm25_idx on {{schema}}.memory using bm25 (content) -with (text_config = '{{bm25_text_config}}', k1 = {{bm25_k1}}, b = {{bm25_b}}); - --- index for vector similarity search -create index memory_embedding_hnsw_idx on {{schema}}.memory using hnsw (embedding halfvec_cosine_ops) -with (m = {{hnsw_m}}, ef_construction = {{hnsw_ef_construction}}); - --- index for hierarchical organization -create index memory_tree_gist_idx on {{schema}}.memory using gist (tree); - --- index for efficiently finding rows with null embeddings -create index memory_null_embedding_idx on {{schema}}.memory (created_at) where (embedding is null and embedding_attempts < 3); - --- make sure the metadata is an object -alter table {{schema}}.memory add check (jsonb_typeof(meta) = 'object'); - -/* -enforce consistent temporal range conventions: -- point-in-time events: lower = upper with inclusive bounds '[same,same]' -- time periods: lower < upper with inclusive-exclusive bounds '[start,end)' -*/ -alter table {{schema}}.memory add constraint temporal_bounds_convention check -( - temporal is null - or ( - -- point-in-time: both bounds equal and inclusive - (lower(temporal) = upper(temporal) and lower_inc(temporal) and upper_inc(temporal)) - or - -- time range: start before end, inclusive-exclusive - (lower(temporal) < upper(temporal) and lower_inc(temporal) and not upper_inc(temporal)) - ) -); diff --git a/packages/engine/migrate/migrations/003_memory_trigger.sql b/packages/engine/migrate/migrations/003_memory_trigger.sql deleted file mode 100644 index 48c5a81..0000000 --- a/packages/engine/migrate/migrations/003_memory_trigger.sql +++ /dev/null @@ -1,34 +0,0 @@ --- before-update trigger for memory table -create function {{schema}}.memory_before_update() -returns trigger -as $func$ -begin - -- always update the timestamp - new.updated_at = pg_catalog.now(); - - -- content changed -> new embedding needs to be generated - if old.content is distinct from new.content - and old.embedding is not distinct from new.embedding - then - new.embedding = null; - new.embedding_attempts = 0; - new.embedding_version = old.embedding_version operator(pg_catalog.+) 1; - new.embedding_last_error = null; - end if; - - -- likely the embedding engine setting the embedding - if new.embedding is not null then - new.embedding_attempts = 0; - new.embedding_last_error = null; - end if; - - return new; -end; -$func$ language plpgsql volatile security definer -set search_path to {{schema}}, public, pg_temp; -- public required for pgvector's `is not distinct from` - -create trigger memory_before_update_trg -before update on {{schema}}.memory -for each row -execute function {{schema}}.memory_before_update(); - diff --git a/packages/engine/migrate/migrations/004_auth_tables.sql b/packages/engine/migrate/migrations/004_auth_tables.sql deleted file mode 100644 index e32fa6b..0000000 --- a/packages/engine/migrate/migrations/004_auth_tables.sql +++ /dev/null @@ -1,227 +0,0 @@ --- ===== Users ===== --- User: thing that accesses memories, or a role (can_login = false) --- identity_id is a soft FK to accounts.identity (nullable for service users) --- Note: "user" is a reserved word, must be quoted -create table {{schema}}."user" -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, name citext not null unique -, identity_id uuid check (identity_id is null or uuid_extract_version(identity_id) = 7) -- soft FK to accounts.identity -, can_login boolean not null default true -- false = role (grant container) -, superuser boolean not null default false -, createrole boolean not null default false -- can create other users/roles -, created_at timestamptz not null default now() -, updated_at timestamptz -); - -create index idx_user_identity_id on {{schema}}."user" (identity_id) where identity_id is not null; - -create trigger user_updated_at - before update on {{schema}}."user" - for each row - execute function {{schema}}.update_updated_at(); - --- ===== API Keys ===== --- Engine-scoped, user-scoped authentication -create table {{schema}}.api_key -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, user_id uuid not null references {{schema}}."user" on delete cascade -, lookup_id text unique not null check (lookup_id ~ '^[A-Za-z0-9_-]{16}$') -, key_hash text not null -, name text not null -, expires_at timestamptz -, created_at timestamptz not null default now() -, revoked_at timestamptz -); - -create index idx_api_key_user on {{schema}}.api_key (user_id); -create index idx_api_key_lookup on {{schema}}.api_key (lookup_id) where revoked_at is null; - --- ===== Tree Grants ===== -create table {{schema}}.tree_grant -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, user_id uuid not null references {{schema}}."user"(id) on delete cascade -, tree_path ltree not null -, actions text[] not null -, granted_by uuid references {{schema}}."user"(id) -, created_at timestamptz not null default now() -, with_grant_option boolean not null default false -, constraint valid_actions check ( - actions <@ '{read,create,update,delete}'::text[] - ) -); - -create unique index idx_tree_grant_unique - on {{schema}}.tree_grant (user_id, tree_path); - -create index idx_tree_grant_user - on {{schema}}.tree_grant using btree (user_id); - -create index idx_tree_grant_path - on {{schema}}.tree_grant using gist (tree_path); - --- ===== Role Membership ===== -create table {{schema}}.role_membership -( role_id uuid not null references {{schema}}."user"(id) on delete cascade -, member_id uuid not null references {{schema}}."user"(id) on delete cascade -, with_admin_option boolean not null default false -, created_at timestamptz not null default now() -, primary key (role_id, member_id) -, constraint no_self_membership check (role_id <> member_id) -); - -create index idx_role_membership_member on {{schema}}.role_membership(member_id); - --- ===== Cycle Detection ===== -create function {{schema}}.would_create_cycle -( _role_id uuid -, _member_id uuid -) -returns boolean -as $func$ - with recursive ancestors(id) as ( - select rm.role_id - from {{schema}}.role_membership rm - where rm.member_id = _role_id - union - select rm.role_id - from {{schema}}.role_membership rm - inner join ancestors a on a.id = rm.member_id - ) - select _member_id = _role_id - or exists - ( - select 1 - from ancestors - where id = _member_id - ) -$func$ language sql stable security invoker -set search_path to pg_catalog, {{schema}}, pg_temp -; - --- ===== Tree Ownership ===== -create table {{schema}}.tree_owner -( tree_path ltree primary key -, user_id uuid not null references {{schema}}."user"(id) on delete cascade -, created_by uuid references {{schema}}."user"(id) -, created_at timestamptz not null default now() -); - -create index idx_tree_owner_user on {{schema}}.tree_owner (user_id); -create index idx_tree_owner_gist on {{schema}}.tree_owner using gist (tree_path); - --- ===== Access Checking (role-aware) ===== --- Returns set of tree paths the user can access for the given action. --- Superusers get ''::ltree (empty root) which matches all paths via <@. - -create function {{schema}}.tree_access -( _user_id uuid -, _action text -) -returns setof ltree -as $func$ - with recursive effective_roles(user_id) as - ( - select _user_id - union - select rm.role_id - from {{schema}}.role_membership rm - inner join effective_roles er on (er.user_id = rm.member_id) - ) - select distinct tree_path - from - ( - -- superuser: empty ltree matches everything via <@ - select ''::ltree as tree_path - from {{schema}}."user" u - inner join effective_roles er on (u.id = er.user_id) - where u.superuser - union - -- ownership grants full access - select o.tree_path - from {{schema}}.tree_owner o - inner join effective_roles er on (er.user_id = o.user_id) - union - -- explicit grants for the requested action - select g.tree_path - from {{schema}}.tree_grant g - inner join effective_roles er on (er.user_id = g.user_id) - where _action = any(g.actions) - ) -$func$ -language sql stable security definer -set search_path to pg_catalog, {{schema}}, public, pg_temp -; - -revoke all on function {{schema}}.tree_access(uuid, text) from public; -grant execute on function {{schema}}.tree_access(uuid, text) to me_ro, me_rw; - --- defense in depth: revoke PUBLIC access on auth tables -revoke all on {{schema}}."user" from public; -revoke all on {{schema}}.api_key from public; -revoke all on {{schema}}.tree_grant from public; -revoke all on {{schema}}.role_membership from public; -revoke all on {{schema}}.tree_owner from public; - --- ===== RLS on memory ===== -alter table {{schema}}.memory enable row level security; - -create policy memory_select on {{schema}}.memory - for select to me_ro, me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'read') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_insert on {{schema}}.memory - for insert to me_rw - with check - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'create') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_update on {{schema}}.memory - for update to me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) - where tree <@ ta.tree_path - ) - ) - with check - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_delete on {{schema}}.memory - for delete to me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'delete') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - --- ===== Memory FK ===== -alter table {{schema}}.memory add constraint memory_created_by_fk - foreign key (created_by) references {{schema}}."user"(id) on delete set null; diff --git a/packages/engine/migrate/migrations/005_embedding_queue.sql b/packages/engine/migrate/migrations/005_embedding_queue.sql deleted file mode 100644 index c325316..0000000 --- a/packages/engine/migrate/migrations/005_embedding_queue.sql +++ /dev/null @@ -1,150 +0,0 @@ --- per-engine embedding queue table -create table {{schema}}.embedding_queue -( id bigint generated always as identity primary key -, memory_id uuid not null references {{schema}}.memory(id) on delete cascade -, embedding_version int not null -, vt timestamptz not null default now() -, outcome text check (outcome is null or outcome in ('completed', 'failed', 'cancelled')) -, attempts int not null default 0 -, max_attempts int not null default 3 -, last_error text -, created_at timestamptz not null default now() -); - -create index embedding_queue_claim_idx - on {{schema}}.embedding_queue (vt) - where outcome is null; -create index embedding_queue_memory_idx - on {{schema}}.embedding_queue (memory_id, embedding_version desc) - where outcome is null; -create index embedding_queue_archive_idx - on {{schema}}.embedding_queue (created_at) - where outcome is not null; - --- enqueue function (SECURITY DEFINER — me_rw cannot access queue directly) -create or replace function {{schema}}.enqueue_embedding() -returns trigger -language plpgsql volatile security definer -set search_path to pg_catalog, {{schema}}, pg_temp -as $func$ -begin - insert into {{schema}}.embedding_queue (memory_id, embedding_version) - values (new.id, new.embedding_version); - return new; -end; -$func$; - --- enqueue triggers -create trigger memory_enqueue_embedding_insert - after insert on {{schema}}.memory - for each row - when (new.embedding is null) - execute function {{schema}}.enqueue_embedding(); - -create trigger memory_enqueue_embedding_update - after update on {{schema}}.memory - for each row - when (old.content is distinct from new.content - and new.embedding is null - and new.embedding_attempts < 3) - execute function {{schema}}.enqueue_embedding(); - --- claim function for embedding worker -create or replace function {{schema}}.claim_embedding_batch( - batch_size int default 10, - lock_duration interval default '5 minutes' -) -returns table (queue_id bigint, memory_id uuid, embedding_version int, content text) -language plpgsql volatile -set search_path to pg_catalog, {{schema}}, pg_temp -as $func$ -declare - rec record; - mem record; - claimed_count int := 0; -begin - -- bulk-cancel visible queue rows superseded by a newer row for the same memory - update {{schema}}.embedding_queue eq - set outcome = 'cancelled' - where eq.outcome is null - and eq.vt <= now() - and exists ( - select 1 - from {{schema}}.embedding_queue newer - where newer.memory_id = eq.memory_id - and newer.embedding_version > eq.embedding_version - and newer.outcome is null - ); - - -- sweep: finalize exhausted rows orphaned by worker crash - -- (attempts reached max but outcome was never written back) - update {{schema}}.embedding_queue - set outcome = 'failed' - , last_error = coalesce(last_error, 'exceeded max attempts (worker crash)') - where outcome is null - and vt <= now() - and attempts >= max_attempts; - - for rec in - select eq.id, eq.memory_id, eq.embedding_version - from {{schema}}.embedding_queue eq - where eq.outcome is null - and eq.vt <= now() - and eq.attempts < eq.max_attempts - order by eq.vt - for update skip locked - loop - -- check memory still exists + current version - select m.content, m.embedding_version - into mem - from {{schema}}.memory m - where m.id = rec.memory_id; - - if not found or mem.content is null then - -- memory deleted or empty → cancel queue row - update {{schema}}.embedding_queue - set outcome = 'cancelled' - where id = rec.id; - continue; - end if; - - if rec.embedding_version <> mem.embedding_version then - -- stale version → cancel - update {{schema}}.embedding_queue - set outcome = 'cancelled' - where id = rec.id; - continue; - end if; - - -- claim this row - update {{schema}}.embedding_queue - set vt = now() + lock_duration - , attempts = {{schema}}.embedding_queue.attempts + 1 - where id = rec.id; - - queue_id := rec.id; - memory_id := rec.memory_id; - embedding_version := rec.embedding_version; - content := mem.content; - return next; - - claimed_count := claimed_count + 1; - exit when claimed_count >= batch_size; - end loop; -end; -$func$; - --- me_embed RLS — system role, unrestricted access to all memories -create policy memory_embed_select on {{schema}}.memory - for select to me_embed - using (true); - -create policy memory_embed_update on {{schema}}.memory - for update to me_embed - using (true); - --- me_embed grants (memory + queue + claim function) -grant usage on schema {{schema}} to me_embed; -grant select, update on {{schema}}.memory to me_embed; -grant select, update, delete on {{schema}}.embedding_queue to me_embed; -grant execute on function {{schema}}.claim_embedding_batch(int, interval) to me_embed; diff --git a/packages/engine/migrate/migrations/006_prune_embedding_queue.sql b/packages/engine/migrate/migrations/006_prune_embedding_queue.sql deleted file mode 100644 index 1a17b08..0000000 --- a/packages/engine/migrate/migrations/006_prune_embedding_queue.sql +++ /dev/null @@ -1,27 +0,0 @@ --- prune terminal queue rows older than the retention window. --- runs opportunistically from the worker on engines that returned no --- claimable work, so the queue table doesn't grow unbounded. --- --- relies on embedding_queue_archive_idx (created_at) where outcome is not null --- from migration 005, so the no-op case is cheap. -create or replace function {{schema}}.prune_embedding_queue -( retention interval default '7 days' -) -returns bigint -language plpgsql volatile -set search_path to pg_catalog, {{schema}}, pg_temp -as $func$ -declare - pruned bigint; -begin - delete from {{schema}}.embedding_queue - where outcome is not null - and created_at < now() - retention; - get diagnostics pruned = row_count; - return pruned; -end; -$func$; - --- me_embed already has DELETE on embedding_queue (granted in 005); --- this just exposes the function entrypoint. -grant execute on function {{schema}}.prune_embedding_queue(interval) to me_embed; diff --git a/packages/engine/migrate/migrations/007_drop_api_key_last_used_at.sql b/packages/engine/migrate/migrations/007_drop_api_key_last_used_at.sql deleted file mode 100644 index 8094573..0000000 --- a/packages/engine/migrate/migrations/007_drop_api_key_last_used_at.sql +++ /dev/null @@ -1,2 +0,0 @@ -alter table {{schema}}.api_key - drop column if exists last_used_at; diff --git a/packages/engine/migrate/migrations/sql.d.ts b/packages/engine/migrate/migrations/sql.d.ts deleted file mode 100644 index 89b092e..0000000 --- a/packages/engine/migrate/migrations/sql.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -declare module "*.sql" { - const content: string; - export default content; -} diff --git a/packages/engine/migrate/provision.ts b/packages/engine/migrate/provision.ts deleted file mode 100644 index d834b6a..0000000 --- a/packages/engine/migrate/provision.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { SQL } from "bun"; - -import { setLocalEngineTimeouts } from "../ops/_tx"; -import { isValidSlug, slugToSchema } from "./discover"; -import { type MigrateResult, migrateEngine } from "./runner"; -import type { EngineConfig } from "./template"; - -export interface ProvisionResult { - schema: string; - migrateResult: MigrateResult; -} - -export async function provisionEngine( - sql: SQL, - slug: string, - config: EngineConfig | undefined, - serverVersion: string, - shardId?: number, -): Promise { - if (!isValidSlug(slug)) { - throw new Error( - `Invalid engine slug: "${slug}" — must be 12 lowercase alphanumeric characters`, - ); - } - - const schema = slugToSchema(slug); - - // Transaction 1: Create schema infrastructure (all or nothing) - await sql.begin(async (tx) => { - if (shardId !== undefined) { - await tx.unsafe(`set local pgdog.shard to ${shardId}`); - } - await setLocalEngineTimeouts(tx); - - // Create schema (fails if exists - use migrateEngine for existing schemas) - await tx.unsafe(`create schema ${schema}`); - - // Version tracking table (single row) - await tx.unsafe(` - create table ${schema}.version - ( version text not null check (version ~ '^\\d+\\.\\d+\\.\\d+$') - , at timestamptz not null default now() - ) - `); - await tx.unsafe(`create unique index on ${schema}.version ((true))`); - await tx.unsafe(`insert into ${schema}.version (version) values ('0.0.0')`); - - // Grant usage to all roles - await tx.unsafe( - `grant usage on schema ${schema} to me_ro, me_rw, me_embed`, - ); - }); - - // Transaction 2: Run migrations - const migrateResult = await migrateEngine( - sql, - schema, - config, - serverVersion, - shardId, - ); - - return { schema, migrateResult }; -} diff --git a/packages/engine/migrate/runner.test.ts b/packages/engine/migrate/runner.test.ts deleted file mode 100644 index aaf355f..0000000 --- a/packages/engine/migrate/runner.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { getMigrations } from "./runner"; - -describe("getMigrations", () => { - test("returns 7 migrations", () => { - expect(getMigrations()).toHaveLength(7); - }); - - test("migrations are sorted by name", () => { - const names = getMigrations().map((m) => m.name); - const sorted = [...names].sort(); - expect(names).toEqual(sorted); - }); - - test("migration names match NNN_name pattern", () => { - for (const { name } of getMigrations()) { - expect(name).toMatch(/^\d{3}_\w+$/); - } - }); - - test("contains expected migration names", () => { - const names = getMigrations().map((m) => m.name); - expect(names).toContain("001_updated_at"); - expect(names).toContain("002_memory"); - expect(names).toContain("003_memory_trigger"); - expect(names).toContain("004_auth_tables"); - expect(names).toContain("005_embedding_queue"); - expect(names).toContain("006_prune_embedding_queue"); - expect(names).toContain("007_drop_api_key_last_used_at"); - }); -}); diff --git a/packages/engine/migrate/runner.ts b/packages/engine/migrate/runner.ts deleted file mode 100644 index c557ccb..0000000 --- a/packages/engine/migrate/runner.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { type SQL, semver } from "bun"; -import { setLocalEngineTimeouts } from "../ops/_tx"; -import { assertEngineSchema } from "./discover"; -import migration001 from "./migrations/001_updated_at.sql" with { - type: "text", -}; -import migration002 from "./migrations/002_memory.sql" with { type: "text" }; -import migration003 from "./migrations/003_memory_trigger.sql" with { - type: "text", -}; -import migration004 from "./migrations/004_auth_tables.sql" with { - type: "text", -}; -import migration005 from "./migrations/005_embedding_queue.sql" with { - type: "text", -}; -import migration006 from "./migrations/006_prune_embedding_queue.sql" with { - type: "text", -}; -import migration007 from "./migrations/007_drop_api_key_last_used_at.sql" with { - type: "text", -}; -import { type EngineConfig, resolveConfig, template } from "./template"; - -interface Migration { - name: string; - sql: string; -} - -const migrations: Migration[] = [ - { name: "001_updated_at", sql: migration001 }, - { name: "002_memory", sql: migration002 }, - { name: "003_memory_trigger", sql: migration003 }, - { name: "004_auth_tables", sql: migration004 }, - { name: "005_embedding_queue", sql: migration005 }, - { name: "006_prune_embedding_queue", sql: migration006 }, - { name: "007_drop_api_key_last_used_at", sql: migration007 }, -]; - -export interface MigrateResult { - schema: string; - status: "ok" | "skipped" | "error"; - applied: string[]; - error?: Error; -} - -const MAX_LOCK_RETRIES = 5; -const BASE_DELAY_MS = 100; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function assertSchemaOwnership(tx: SQL, schema: string): Promise { - const [result] = await tx` - select - n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner - from pg_catalog.pg_namespace n - where n.nspname = ${schema} - `; - - if (!result?.is_owner) { - throw new Error( - `Only the owner of the ${schema} schema can run database migrations`, - ); - } -} - -export async function migrateEngine( - sql: SQL, - schema: string, - config: EngineConfig | undefined, - serverVersion: string, - shardId?: number, -): Promise { - const resolved = resolveConfig(schema, config); - - return await sql.begin(async (tx) => { - if (shardId !== undefined) { - await tx.unsafe(`set local pgdog.shard to ${shardId}`); - } - await setLocalEngineTimeouts(tx); - - await assertEngineSchema(tx, schema); - - // 1. Acquire advisory lock with retry - const [{ lock_id }] = await tx` - select hashtext(${schema})::bigint as lock_id - `; - - let acquired = false; - for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { - const [result] = await tx` - select pg_try_advisory_xact_lock(${lock_id}) as acquired - `; - if (result.acquired) { - acquired = true; - break; - } - if (attempt < MAX_LOCK_RETRIES - 1) { - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } - - if (!acquired) { - return { schema, status: "skipped" as const, applied: [] }; - } - - // 2. Check ownership - await assertSchemaOwnership(tx, schema); - - // 3. Check version (reject downgrades) - const [{ version: dbVersion }] = await tx.unsafe( - `select version from ${schema}.version`, - ); - - const cmp = semver.order(serverVersion, dbVersion); - if (cmp < 0) { - throw new Error( - `Server version (${serverVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the server.", - ); - } - - // 4. Scaffold migration tracking table - await tx.unsafe(` - create table if not exists ${schema}.migration - ( name text not null primary key - , applied_at_version text not null - , applied_at timestamptz not null default pg_catalog.clock_timestamp() - ) - `); - - // 5. Run migrations - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - const applied: string[] = []; - - for (const migration of sorted) { - const [existing] = await tx.unsafe( - `select 1 from ${schema}.migration where name = $1`, - [migration.name], - ); - - if (existing) { - continue; - } - - const renderedSql = template(migration.sql, resolved); - await tx.unsafe(renderedSql); - await tx.unsafe( - `insert into ${schema}.migration (name, applied_at_version) values ($1, $2)`, - [migration.name, serverVersion], - ); - applied.push(migration.name); - } - - // 6. Update version if app version is newer - if (cmp > 0) { - await tx.unsafe(`update ${schema}.version set version = $1, at = now()`, [ - serverVersion, - ]); - } - - return { schema, status: "ok" as const, applied }; - }); -} - -export async function migrateAll( - sql: SQL, - schemas: string[], - config: EngineConfig | undefined, - serverVersion: string, - options?: { concurrency?: number; shardId?: number }, -): Promise> { - const concurrency = options?.concurrency ?? 10; - const shardId = options?.shardId; - const results = new Map(); - - // Simple semaphore for bounded parallelism - let active = 0; - let idx = 0; - - const runOne = async (schema: string): Promise => { - try { - const result = await migrateEngine( - sql, - schema, - config, - serverVersion, - shardId, - ); - results.set(schema, result); - } catch (error) { - results.set(schema, { - schema, - status: "error", - applied: [], - error: error instanceof Error ? error : new Error(String(error)), - }); - } - }; - - await new Promise((resolve) => { - if (schemas.length === 0) { - resolve(); - return; - } - - let completed = 0; - - const next = () => { - while (active < concurrency && idx < schemas.length) { - const schema = schemas[idx++]; - if (!schema) break; - active++; - runOne(schema).then(() => { - active--; - completed++; - if (completed === schemas.length) { - resolve(); - } else { - next(); - } - }); - } - }; - - next(); - }); - - return results; -} - -export async function dryRun( - sql: SQL, - schema: string, - _config?: EngineConfig, -): Promise<{ pending: string[]; applied: string[] }> { - await assertEngineSchema(sql, schema); - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - - // Check if migration table exists - const [{ exists }] = await sql` - select exists ( - select 1 - from information_schema.tables - where table_schema = ${schema} - and table_name = 'migration' - ) as exists - `; - - if (!exists) { - return { - pending: sorted.map((m) => m.name), - applied: [], - }; - } - - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - const appliedSet = new Set(rows.map((r: { name: string }) => r.name)); - const applied = sorted - .filter((m) => appliedSet.has(m.name)) - .map((m) => m.name); - const pending = sorted - .filter((m) => !appliedSet.has(m.name)) - .map((m) => m.name); - - return { pending, applied }; -} - -export function getMigrations(): ReadonlyArray<{ name: string }> { - return [...migrations] - .sort((a, b) => a.name.localeCompare(b.name)) - .map(({ name }) => ({ name })); -} - -export async function getVersion(sql: SQL, schema: string): Promise { - await assertEngineSchema(sql, schema); - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row.version; -} diff --git a/packages/engine/migrate/template.test.ts b/packages/engine/migrate/template.test.ts deleted file mode 100644 index b657622..0000000 --- a/packages/engine/migrate/template.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { defaultConfig, resolveConfig, template } from "./template"; - -describe("template function", () => { - test("replaces single variable", () => { - const sql = "CREATE TABLE foo (id {{type}})"; - const result = template(sql, { type: "UUID" }); - expect(result).toBe("CREATE TABLE foo (id UUID)"); - }); - - test("replaces multiple variables", () => { - const sql = "CREATE INDEX ON table USING {{method}} WITH (m = {{m}})"; - const result = template(sql, { method: "hnsw", m: 16 }); - expect(result).toBe("CREATE INDEX ON table USING hnsw WITH (m = 16)"); - }); - - test("replaces same variable multiple times", () => { - const sql = "{{var}} and {{var}} and {{var}}"; - const result = template(sql, { var: "test" }); - expect(result).toBe("test and test and test"); - }); - - test("throws on missing variable", () => { - const sql = "CREATE TABLE foo (x {{missing}})"; - expect(() => template(sql, {})).toThrow( - "Missing template variable: missing", - ); - }); - - test("handles numeric values", () => { - const sql = "WITH (m = {{value}})"; - const result = template(sql, { value: 1536 }); - expect(result).toBe("WITH (m = 1536)"); - }); - - test("handles decimal values", () => { - const sql = "WITH (k1 = {{k1}}, b = {{b}})"; - const result = template(sql, { k1: 1.2, b: 0.75 }); - expect(result).toBe("WITH (k1 = 1.2, b = 0.75)"); - }); - - test("handles boolean values", () => { - const sql = "SET enabled = {{enabled}}"; - const result = template(sql, { enabled: true }); - expect(result).toBe("SET enabled = true"); - }); - - test("handles empty string values", () => { - const sql = "SET value = '{{value}}'"; - const result = template(sql, { value: "" }); - expect(result).toBe("SET value = ''"); - }); - - test("handles variables with underscores", () => { - const sql = "halfvec({{embedding_dimensions}})"; - const result = template(sql, { embedding_dimensions: 768 }); - expect(result).toBe("halfvec(768)"); - }); - - test("handles variables with numbers", () => { - const sql = "WITH (k1 = {{bm25_k1}})"; - const result = template(sql, { bm25_k1: 1.2 }); - expect(result).toBe("WITH (k1 = 1.2)"); - }); - - test("preserves text outside of variables", () => { - const sql = "CREATE INDEX idx ON table USING {{method}} (column)"; - const result = template(sql, { method: "btree" }); - expect(result).toBe("CREATE INDEX idx ON table USING btree (column)"); - }); - - test("handles variables at start of string", () => { - const sql = "{{type}} NOT NULL"; - const result = template(sql, { type: "UUID" }); - expect(result).toBe("UUID NOT NULL"); - }); - - test("handles variables at end of string", () => { - const sql = "CREATE TYPE {{type}}"; - const result = template(sql, { type: "custom" }); - expect(result).toBe("CREATE TYPE custom"); - }); - - test("handles no variables", () => { - const sql = "CREATE TABLE foo (id UUID)"; - const result = template(sql, {}); - expect(result).toBe("CREATE TABLE foo (id UUID)"); - }); - - test("handles real migration template", () => { - const sql = "halfvec({{embedding_dimensions}})"; - const result = template(sql, { embedding_dimensions: 1536 }); - expect(result).toBe("halfvec(1536)"); - }); - - test("handles BM25 index template", () => { - const sql = - "with (text_config = '{{bm25_text_config}}', k1 = {{bm25_k1}}, b = {{bm25_b}})"; - const result = template(sql, { - bm25_text_config: "english", - bm25_k1: 1.2, - bm25_b: 0.75, - }); - expect(result).toBe("with (text_config = 'english', k1 = 1.2, b = 0.75)"); - }); - - test("handles HNSW index template", () => { - const sql = - "with (m = {{hnsw_m}}, ef_construction = {{hnsw_ef_construction}})"; - const result = template(sql, { - hnsw_m: 16, - hnsw_ef_construction: 64, - }); - expect(result).toBe("with (m = 16, ef_construction = 64)"); - }); - - test("handles schema variable substitution", () => { - const sql = "CREATE TABLE {{schema}}.memory (id uuid)"; - const result = template(sql, { schema: "me_abc123def456" }); - expect(result).toBe("CREATE TABLE me_abc123def456.memory (id uuid)"); - }); -}); - -describe("config merging", () => { - test("merging empty config uses defaults", () => { - const resolved = resolveConfig("me_test123test"); - expect(resolved.embedding_dimensions).toBe(1536); - expect(resolved.bm25_text_config).toBe("english"); - expect(resolved.bm25_k1).toBe(1.2); - expect(resolved.bm25_b).toBe(0.75); - expect(resolved.hnsw_m).toBe(16); - expect(resolved.hnsw_ef_construction).toBe(64); - expect(resolved.schema).toBe("me_test123test"); - }); - - test("partial override only changes specified values", () => { - const resolved = resolveConfig("me_test123test", { - embedding_dimensions: 768, - }); - expect(resolved.embedding_dimensions).toBe(768); - expect(resolved.bm25_text_config).toBe("english"); - }); - - test("multiple overrides work correctly", () => { - const resolved = resolveConfig("me_test123test", { - embedding_dimensions: 384, - bm25_text_config: "simple", - hnsw_m: 32, - }); - expect(resolved.embedding_dimensions).toBe(384); - expect(resolved.bm25_text_config).toBe("simple"); - expect(resolved.hnsw_m).toBe(32); - expect(resolved.bm25_k1).toBe(1.2); - expect(resolved.bm25_b).toBe(0.75); - }); - - test("numeric config values preserved as numbers", () => { - const resolved = resolveConfig("me_test123test", { - bm25_k1: 2.5, - bm25_b: 0.9, - }); - expect(typeof resolved.bm25_k1).toBe("number"); - expect(typeof resolved.bm25_b).toBe("number"); - expect(resolved.bm25_k1).toBe(2.5); - expect(resolved.bm25_b).toBe(0.9); - }); - - test("schema is always set from argument", () => { - const resolved = resolveConfig("me_abc123def456", { - embedding_dimensions: 768, - }); - expect(resolved.schema).toBe("me_abc123def456"); - }); -}); - -describe("defaultConfig", () => { - test("has all required fields", () => { - expect(defaultConfig.embedding_dimensions).toBe(1536); - expect(defaultConfig.bm25_text_config).toBe("english"); - expect(defaultConfig.bm25_k1).toBe(1.2); - expect(defaultConfig.bm25_b).toBe(0.75); - expect(defaultConfig.hnsw_m).toBe(16); - expect(defaultConfig.hnsw_ef_construction).toBe(64); - }); -}); diff --git a/packages/engine/migrate/template.ts b/packages/engine/migrate/template.ts deleted file mode 100644 index f3ffa1d..0000000 --- a/packages/engine/migrate/template.ts +++ /dev/null @@ -1,38 +0,0 @@ -export function template(sql: string, vars: Record): string { - return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { - if (!(key in vars)) { - throw new Error(`Missing template variable: ${key}`); - } - return String(vars[key]); - }); -} - -// Global index/search configuration — same for all engines in a database. -// Schema is not included here because it's per-engine, not per-database. -export interface EngineConfig { - embedding_dimensions?: number; - bm25_text_config?: string; - bm25_k1?: number; - bm25_b?: number; - hnsw_m?: number; - hnsw_ef_construction?: number; -} - -// All defaults filled in + per-engine schema attached. Used internally by template(). -export type ResolvedConfig = Required & { schema: string }; - -export const defaultConfig: Required = { - embedding_dimensions: 1536, - bm25_text_config: "english", - bm25_k1: 1.2, - bm25_b: 0.75, - hnsw_m: 16, - hnsw_ef_construction: 64, -}; - -export function resolveConfig( - schema: string, - config?: EngineConfig, -): ResolvedConfig { - return { ...defaultConfig, ...config, schema }; -} diff --git a/packages/engine/migrate/test-utils.ts b/packages/engine/migrate/test-utils.ts deleted file mode 100644 index a24648f..0000000 --- a/packages/engine/migrate/test-utils.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { SQL } from "bun"; - -function assertSafeIdentifier(name: string): void { - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { - throw new Error(`Unsafe database identifier: ${name}`); - } -} - -export class TestDatabase { - private dbName: string | null = null; - private readonly adminUrl: string; - - constructor(adminUrl = "postgresql://postgres@localhost:5432/postgres") { - this.adminUrl = adminUrl; - } - - async create(): Promise { - this.dbName = `test_me_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - - assertSafeIdentifier(this.dbName); - const sql = new SQL(this.adminUrl); - try { - await sql.unsafe(`create database ${this.dbName}`); - } finally { - await sql.close(); - } - - const url = new URL(this.adminUrl); - url.pathname = `/${this.dbName}`; - return url.toString(); - } - - async drop(): Promise { - if (!this.dbName) { - return; - } - - assertSafeIdentifier(this.dbName); - const sql = new SQL(this.adminUrl); - try { - await sql` - select pg_terminate_backend(pg_stat_activity.pid) - from pg_stat_activity - where pg_stat_activity.datname = ${this.dbName} - and pid <> pg_backend_pid() - `; - - await sql.unsafe(`drop database if exists ${this.dbName}`); - } finally { - await sql.close(); - this.dbName = null; - } - } -} - -export async function getAppliedMigrations( - sql: SQL, - schema: string, -): Promise { - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - return rows.map((r: { name: string }) => r.name); -} - -export async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 - from information_schema.tables - where table_schema = ${schema} - and table_name = ${table} - ) as exists - `; - return row.exists; -} - -export async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 - from information_schema.schemata - where schema_name = ${name} - ) as exists - `; - return row.exists; -} - -export async function countMigrations( - sql: SQL, - schema: string, -): Promise { - const [row] = await sql.unsafe( - `select count(*)::int as count from ${schema}.migration`, - ); - return row.count; -} - -export async function getTableColumns( - sql: SQL, - schema: string, - table: string, -): Promise< - Array<{ column_name: string; data_type: string; is_nullable: string }> -> { - return await sql` - select column_name, data_type, is_nullable - from information_schema.columns - where table_schema = ${schema} - and table_name = ${table} - order by ordinal_position - `; -} - -export async function getIndexes( - sql: SQL, - schema: string, - table: string, -): Promise { - const rows = await sql` - select indexname - from pg_indexes - where schemaname = ${schema} - and tablename = ${table} - order by indexname - `; - return rows.map((r: { indexname: string }) => r.indexname); -} - -export async function getRoles( - sql: SQL, - ...names: string[] -): Promise> { - const pgArray = `{${names.join(",")}}`; - return await sql.unsafe( - `select rolname, rolcanlogin - from pg_roles - where rolname = any($1::text[]) - order by rolname`, - [pgArray], - ); -} - -export async function getFunctions( - sql: SQL, - schema: string, -): Promise> { - return await sql` - select p.proname, p.proconfig - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = ${schema} - order by p.proname - `; -} diff --git a/packages/engine/ops/_tx.ts b/packages/engine/ops/_tx.ts deleted file mode 100644 index a62e0a2..0000000 --- a/packages/engine/ops/_tx.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { span } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import type { OpsContext } from "../types"; - -/** - * Transaction modes: - * - "read": Sets ROLE me_ro (read-only, RLS enforced) - * - "write": Sets ROLE me_rw (read-write, RLS enforced) - * - "admin": No role change (runs as connection owner, bypasses RLS) - * - * Admin mode is used for auth operations (principal, api_key, grant, etc.) - * which are not protected by RLS — only the memory table has RLS policies. - */ -type TransactionMode = "read" | "write" | "admin"; - -const ROLE_MAP = { - read: "me_ro", - write: "me_rw", - admin: null, // No role change -} as const; - -export interface EngineTimeouts { - statementTimeout: string; - lockTimeout: string; - transactionTimeout: string; - idleInTransactionSessionTimeout: string; -} - -export const DEFAULT_ENGINE_TIMEOUTS: EngineTimeouts = { - statementTimeout: process.env.ENGINE_STATEMENT_TIMEOUT ?? "25s", - lockTimeout: process.env.ENGINE_LOCK_TIMEOUT ?? "5s", - transactionTimeout: process.env.ENGINE_TRANSACTION_TIMEOUT ?? "30s", - idleInTransactionSessionTimeout: - process.env.ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? "30s", -}; - -/** - * Bound engine queries so production failures surface before clients give up. - * Uses transaction-local GUCs so pooled connections do not retain settings. - */ -export async function setLocalEngineTimeouts( - sql: SQL, - timeouts: EngineTimeouts = DEFAULT_ENGINE_TIMEOUTS, -): Promise { - await sql.unsafe("SELECT set_config('statement_timeout', $1, true)", [ - timeouts.statementTimeout, - ]); - await sql.unsafe("SELECT set_config('lock_timeout', $1, true)", [ - timeouts.lockTimeout, - ]); - await sql.unsafe("SELECT set_config('transaction_timeout', $1, true)", [ - timeouts.transactionTimeout, - ]); - await sql.unsafe( - "SELECT set_config('idle_in_transaction_session_timeout', $1, true)", - [timeouts.idleInTransactionSessionTimeout], - ); -} - -/** - * Execute a function within a transaction context. - * - * If already inside a transaction (ctx.inTransaction is true), just sets - * the appropriate role and runs the function on the existing handle. - * - * If not inside a transaction, opens a new one with: - * - SET LOCAL pgdog.shard (if shard is set, for future sharding) - * - SET LOCAL statement_timeout / lock_timeout / transaction timeouts - * - SET LOCAL search_path (insurance — all SQL is schema-qualified) - * - SET LOCAL ROLE (me_ro for read, me_rw for write) - * - set_config('me.user_id', ...) (for RLS) - */ -export async function withTx( - ctx: OpsContext, - mode: TransactionMode, - operation: string, - fn: (sql: SQL) => Promise, -): Promise { - const role = ROLE_MAP[mode]; - - if (ctx.inTransaction) { - // Already in a transaction — set role (if not admin) and run directly - if (role) { - await ctx.sql.unsafe(`SET LOCAL ROLE ${role}`); - } - return fn(ctx.sql); - } - - // Open new transaction with telemetry span - return span(`db.${operation}`, { - attributes: { - "db.schema": ctx.schema, - "db.mode": mode, - "db.role": role ?? "owner", - "db.operation": operation, - }, - callback: () => - ctx.sql.begin(async (tx) => { - // Future: pgDog shard routing - if (ctx.shard !== undefined) { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${ctx.shard}`); - } - - await setLocalEngineTimeouts(tx); - - // Set search_path: engine schema first, then public (for extension types like ltree) - await tx.unsafe(`SET LOCAL search_path TO ${ctx.schema}, public`); - - // Set role for permission control (skip for admin mode) - if (role) { - await tx.unsafe(`SET LOCAL ROLE ${role}`); - } - - // Set user_id for RLS policies (only meaningful for read/write modes) - const userId = ctx.getUserId(); - if (userId && role) { - await tx`SELECT set_config('me.user_id', ${userId}, true)`; - } - - return fn(tx); - }), - }); -} - -/** - * Helper to create a derived OpsContext for use inside withTransaction() - */ -export function deriveContext(ctx: OpsContext, tx: SQL): OpsContext { - return { - ...ctx, - sql: tx, - inTransaction: true, - }; -} diff --git a/packages/engine/ops/api-key.ts b/packages/engine/ops/api-key.ts deleted file mode 100644 index 68843a7..0000000 --- a/packages/engine/ops/api-key.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { span } from "@pydantic/logfire-node"; -import type { - ApiKey, - CreateApiKeyParams, - CreateApiKeyResult, - OpsContext, - ValidateApiKeyResult, -} from "../types"; -import { - formatApiKey, - generateLookupId, - generateSecret, - hashSecret, - verifySecret, -} from "../util/api-key"; -import { withTx } from "./_tx"; - -// Row type from database -interface ApiKeyRow { - id: string; - user_id: string; - lookup_id: string; - key_hash: string; - name: string; - expires_at: Date | null; - created_at: Date; - revoked_at: Date | null; -} - -function rowToApiKey(row: ApiKeyRow): ApiKey { - return { - id: row.id, - userId: row.user_id, - lookupId: row.lookup_id, - name: row.name, - expiresAt: row.expires_at, - createdAt: row.created_at, - revokedAt: row.revoked_at, - }; -} - -export function apiKeyOps(ctx: OpsContext, engineSlug: string) { - const { schema } = ctx; - - return { - /** - * Create a new API key for a user - * Returns the full key string (only available at creation time) - */ - async createApiKey( - params: CreateApiKeyParams, - ): Promise { - const { userId, name, expiresAt = null } = params; - - const lookupId = generateLookupId(); - const secret = generateSecret(); - const keyHash = await hashSecret(secret); - const rawKey = formatApiKey(engineSlug, lookupId, secret); - - return withTx(ctx, "admin", "createApiKey", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.api_key - (user_id, lookup_id, key_hash, name, expires_at) - values - (${userId}, ${lookupId}, ${keyHash}, ${name}, ${expiresAt}) - returning id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create API key"); - } - return { - apiKey: rowToApiKey(row), - rawKey, - }; - }); - }, - - /** - * Validate an API key and return the user ID if valid - */ - async validateApiKey( - lookupId: string, - secret: string, - ): Promise { - return withTx(ctx, "admin", "validateApiKey", async (sql) => { - const row = await span("db.api_key.lookup", { - attributes: { - "db.schema": schema, - "engine.slug": engineSlug, - "api_key.lookup_id": lookupId, - }, - callback: async () => { - const [apiKey] = await sql` - select id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - from ${sql.unsafe(schema)}.api_key - where lookup_id = ${lookupId} - `; - return apiKey; - }, - }); - - if (!row) { - return { valid: false, error: "API key not found" }; - } - - if (row.revoked_at) { - return { valid: false, error: "API key has been revoked" }; - } - - if (row.expires_at && row.expires_at < new Date()) { - return { valid: false, error: "API key has expired" }; - } - - const secretValid = await span("auth.api_key.verify_secret", { - attributes: { - "db.schema": schema, - "engine.slug": engineSlug, - "api_key.id": row.id, - "api_key.lookup_id": lookupId, - }, - callback: () => verifySecret(secret, row.key_hash), - }); - if (!secretValid) { - return { valid: false, error: "Invalid API key secret" }; - } - - return { - valid: true, - userId: row.user_id, - apiKeyId: row.id, - }; - }); - }, - - /** - * Get an API key by ID (without the secret/hash) - */ - async getApiKey(id: string): Promise { - return withTx(ctx, "admin", "getApiKey", async (sql) => { - const [row] = await sql` - select id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - from ${sql.unsafe(schema)}.api_key - where id = ${id} - `; - return row ? rowToApiKey(row) : null; - }); - }, - - /** - * List all API keys for a user - */ - async listApiKeys(userId: string): Promise { - return withTx(ctx, "admin", "listApiKeys", async (sql) => { - const rows = await sql` - select id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - from ${sql.unsafe(schema)}.api_key - where user_id = ${userId} - order by created_at desc - `; - return rows.map(rowToApiKey); - }); - }, - - /** - * Revoke an API key - */ - async revokeApiKey(id: string): Promise { - return withTx(ctx, "admin", "revokeApiKey", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.api_key - set revoked_at = now() - where id = ${id} - and revoked_at is null - `; - return result.count > 0; - }); - }, - - /** - * Delete an API key permanently - */ - async deleteApiKey(id: string): Promise { - return withTx(ctx, "admin", "deleteApiKey", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.api_key - where id = ${id} - `; - return result.count > 0; - }); - }, - }; -} - -export type ApiKeyOps = ReturnType; diff --git a/packages/engine/ops/grant.ts b/packages/engine/ops/grant.ts deleted file mode 100644 index a783afb..0000000 --- a/packages/engine/ops/grant.ts +++ /dev/null @@ -1,172 +0,0 @@ -import type { GrantTreeAccessParams, OpsContext, TreeGrant } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface TreeGrantRow { - id: string; - user_id: string; - user_name: string; - tree_path: string; - actions: string[]; - granted_by: string | null; - with_grant_option: boolean; - created_at: Date; -} - -function rowToTreeGrant(row: TreeGrantRow): TreeGrant { - return { - id: row.id, - userId: row.user_id, - userName: row.user_name, - treePath: row.tree_path, - actions: row.actions, - grantedBy: row.granted_by, - withGrantOption: row.with_grant_option, - createdAt: row.created_at, - }; -} - -export function grantOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Grant tree access to a user (upserts on user_id + tree_path) - */ - async grantTreeAccess(params: GrantTreeAccessParams): Promise { - const { - userId, - treePath, - actions, - grantedBy = null, - withGrantOption = false, - } = params; - - await withTx(ctx, "admin", "grantTreeAccess", async (sql) => { - // Format actions as PostgreSQL array literal - const actionsArray = `{${actions.join(",")}}`; - await sql` - insert into ${sql.unsafe(schema)}.tree_grant - (user_id, tree_path, actions, granted_by, with_grant_option) - values - (${userId}, ${treePath}::ltree, ${actionsArray}::text[], ${grantedBy}, ${withGrantOption}) - on conflict (user_id, tree_path) - do update set - actions = excluded.actions, - granted_by = excluded.granted_by, - with_grant_option = excluded.with_grant_option - `; - }); - }, - - /** - * Revoke tree access from a user - */ - async revokeTreeAccess(userId: string, treePath: string): Promise { - return withTx(ctx, "admin", "revokeTreeAccess", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.tree_grant - where user_id = ${userId} - and tree_path = ${treePath}::ltree - `; - return result.count > 0; - }); - }, - - /** - * List tree grants, optionally filtered by user - */ - async listTreeGrants(userId?: string): Promise { - return withTx(ctx, "admin", "listTreeGrants", async (sql) => { - if (userId) { - const rows = await sql` - select g.id, g.user_id, u.name as user_name, g.tree_path::text, g.actions, g.granted_by, g.with_grant_option, g.created_at - from ${sql.unsafe(schema)}.tree_grant g - join ${sql.unsafe(schema)}."user" u on u.id = g.user_id - where g.user_id = ${userId} - order by g.tree_path - `; - return rows.map(rowToTreeGrant); - } - - const rows = await sql` - select g.id, g.user_id, u.name as user_name, g.tree_path::text, g.actions, g.granted_by, g.with_grant_option, g.created_at - from ${sql.unsafe(schema)}.tree_grant g - join ${sql.unsafe(schema)}."user" u on u.id = g.user_id - order by u.name, g.tree_path - `; - return rows.map(rowToTreeGrant); - }); - }, - - /** - * Get a specific grant by user and tree path - */ - async getTreeGrant( - userId: string, - treePath: string, - ): Promise { - return withTx(ctx, "admin", "getTreeGrant", async (sql) => { - const rows = await sql` - select g.id, g.user_id, u.name as user_name, g.tree_path::text, g.actions, g.granted_by, g.with_grant_option, g.created_at - from ${sql.unsafe(schema)}.tree_grant g - join ${sql.unsafe(schema)}."user" u on u.id = g.user_id - where g.user_id = ${userId} - and g.tree_path = ${treePath}::ltree - `; - const row = rows[0]; - return row ? rowToTreeGrant(row) : null; - }); - }, - - /** - * Check if a user has access to a tree path for a given action - * Uses the database's tree_access function (includes role inheritance) - */ - async checkTreeAccess( - userId: string, - treePath: string, - action: string, - ): Promise { - return withTx(ctx, "admin", "checkTreeAccess", async (sql) => { - const rows = await sql<{ allowed: boolean }[]>` - select exists( - select 1 - from ${sql.unsafe(schema)}.tree_access( - ${userId}::uuid, - ${action} - ) ta(tree_path) - where ${treePath}::ltree <@ ta.tree_path - ) as allowed - `; - return rows[0]?.allowed ?? false; - }); - }, - - /** - * Check if a user has grant option for a tree path and actions - */ - async hasGrantOption( - userId: string, - treePath: string, - actions: string[], - ): Promise { - return withTx(ctx, "admin", "hasGrantOption", async (sql) => { - const actionsArray = `{${actions.join(",")}}`; - const rows = await sql<{ has_option: boolean }[]>` - select exists ( - select 1 - from ${sql.unsafe(schema)}.tree_grant - where user_id = ${userId} - and ${treePath}::ltree <@ tree_path - and with_grant_option = true - and actions @> ${actionsArray}::text[] - ) as has_option - `; - return rows[0]?.has_option ?? false; - }); - }, - }; -} - -export type GrantOps = ReturnType; diff --git a/packages/engine/ops/index.ts b/packages/engine/ops/index.ts deleted file mode 100644 index 5ab1f57..0000000 --- a/packages/engine/ops/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { deriveContext, withTx } from "./_tx"; -export { type ApiKeyOps, apiKeyOps } from "./api-key"; -export { type GrantOps, grantOps } from "./grant"; -export { type MemoryOps, memoryOps } from "./memory"; -export { type OwnerOps, ownerOps } from "./owner"; -export { type RoleOps, roleOps } from "./role"; -export { type UserOps, userOps } from "./user"; diff --git a/packages/engine/ops/memory.test.ts b/packages/engine/ops/memory.test.ts deleted file mode 100644 index 9a69101..0000000 --- a/packages/engine/ops/memory.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { detectTreeFilterType, rrfFusion } from "./memory"; - -describe("detectTreeFilterType", () => { - test("detects ltxtquery (contains &)", () => { - expect(detectTreeFilterType("api & v2")).toBe("ltxtquery"); - expect(detectTreeFilterType("work & !draft")).toBe("ltxtquery"); - expect(detectTreeFilterType("a & b & c")).toBe("ltxtquery"); - }); - - test("detects lquery (contains pattern chars)", () => { - // Wildcard - expect(detectTreeFilterType("work.*")).toBe("lquery"); - expect(detectTreeFilterType("*.api.*")).toBe("lquery"); - - // Quantifier - expect(detectTreeFilterType("work.*{2}")).toBe("lquery"); - expect(detectTreeFilterType("work.*{1,3}")).toBe("lquery"); - - // Negation - expect(detectTreeFilterType("*.!draft.*")).toBe("lquery"); - - // Alternation - expect(detectTreeFilterType("work|personal.*")).toBe("lquery"); - - // Other pattern chars - expect(detectTreeFilterType("work.@api")).toBe("lquery"); - expect(detectTreeFilterType("work.%")).toBe("lquery"); - }); - - test("detects ltree (plain paths)", () => { - expect(detectTreeFilterType("work")).toBe("ltree"); - expect(detectTreeFilterType("work.projects")).toBe("ltree"); - expect(detectTreeFilterType("work.projects.api")).toBe("ltree"); - expect(detectTreeFilterType("a.b.c.d.e")).toBe("ltree"); - expect(detectTreeFilterType("")).toBe("ltree"); - }); -}); - -describe("rrfFusion", () => { - test("combines results from both sources", () => { - const bm25Results = [{ id: "a" }, { id: "b" }, { id: "c" }]; - const semanticResults = [{ id: "b" }, { id: "d" }, { id: "a" }]; - - const fused = rrfFusion(bm25Results, semanticResults); - - // All unique IDs should be present - const ids = fused.map((r) => r.id); - expect(ids).toContain("a"); - expect(ids).toContain("b"); - expect(ids).toContain("c"); - expect(ids).toContain("d"); - expect(ids).toHaveLength(4); - }); - - test("ranks items appearing in both lists higher", () => { - const bm25Results = [{ id: "a" }, { id: "b" }]; - const semanticResults = [{ id: "b" }, { id: "c" }]; - - const fused = rrfFusion(bm25Results, semanticResults); - - // 'b' appears in both, should have highest score - expect(fused[0]!.id).toBe("b"); - }); - - test("respects rank order within each list", () => { - const bm25Results = [{ id: "a" }, { id: "b" }, { id: "c" }]; - const semanticResults: Array<{ id: string }> = []; - - const fused = rrfFusion(bm25Results, semanticResults); - - // Order should be preserved from BM25 - expect(fused[0]!.id).toBe("a"); - expect(fused[1]!.id).toBe("b"); - expect(fused[2]!.id).toBe("c"); - }); - - test("applies weights correctly", () => { - const bm25Results = [{ id: "a" }]; - const semanticResults = [{ id: "b" }]; - - // With equal weights - const fusedEqual = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: 1.0, - semantic: 1.0, - }); - expect(fusedEqual[0]!.score).toBe(fusedEqual[1]!.score); - - // With higher BM25 weight - const fusedBM25Heavy = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: 2.0, - semantic: 1.0, - }); - const aScore = fusedBM25Heavy.find((r) => r.id === "a")!.score; - const bScore = fusedBM25Heavy.find((r) => r.id === "b")!.score; - expect(aScore).toBeGreaterThan(bScore); - - // With higher semantic weight - const fusedSemanticHeavy = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: 1.0, - semantic: 2.0, - }); - const aScore2 = fusedSemanticHeavy.find((r) => r.id === "a")!.score; - const bScore2 = fusedSemanticHeavy.find((r) => r.id === "b")!.score; - expect(bScore2).toBeGreaterThan(aScore2); - }); - - test("handles empty inputs", () => { - expect(rrfFusion([], [])).toEqual([]); - expect(rrfFusion([{ id: "a" }], [])).toHaveLength(1); - expect(rrfFusion([], [{ id: "b" }])).toHaveLength(1); - }); - - test("uses k parameter correctly", () => { - const bm25Results = [{ id: "a" }]; - const semanticResults: Array<{ id: string }> = []; - - // RRF score = weight / (k + rank) - // With k=60, rank=1: score = 1 / 61 ≈ 0.0164 - const fusedK60 = rrfFusion(bm25Results, semanticResults, 60); - expect(fusedK60[0]!.score).toBeCloseTo(1 / 61, 5); - - // With k=10, rank=1: score = 1 / 11 ≈ 0.0909 - const fusedK10 = rrfFusion(bm25Results, semanticResults, 10); - expect(fusedK10[0]!.score).toBeCloseTo(1 / 11, 5); - }); -}); diff --git a/packages/engine/ops/memory.ts b/packages/engine/ops/memory.ts deleted file mode 100644 index fa30083..0000000 --- a/packages/engine/ops/memory.ts +++ /dev/null @@ -1,778 +0,0 @@ -import type { SQL } from "bun"; -import type { - CreateMemoryParams, - GetTreeParams, - Memory, - OpsContext, - SearchParams, - SearchResult, - SearchResultItem, - TemporalFilter, - TreeNode, - UpdateMemoryParams, -} from "../types"; -import { withTx } from "./_tx"; - -// ============================================================================= -// Row Types -// ============================================================================= - -interface MemoryRow { - id: string; - content: string; - meta: Record; - tree: string; - temporal: string | null; - has_embedding: boolean; - created_at: Date; - created_by: string | null; - updated_at: Date | null; -} - -interface SearchRow extends MemoryRow { - score: number | string; // Postgres may return as string -} - -interface TreeRow { - path: string; - count: number; -} - -// ============================================================================= -// Tree Filter Detection -// ============================================================================= - -type TreeFilterType = "ltree" | "lquery" | "ltxtquery"; - -/** - * Detect the type of tree filter based on the pattern. - * - ltxtquery: Contains & (label search with AND) - * - lquery: Contains pattern characters (*, {}, !, |, @, %) - * - ltree: Plain path (default) - */ -function detectTreeFilterType(value: string): TreeFilterType { - if (value.includes("&")) return "ltxtquery"; - if (/[*{}!|@%]/.test(value)) return "lquery"; - return "ltree"; -} - -// ============================================================================= -// RRF Fusion -// ============================================================================= - -interface RankedResult { - id: string; - score: number; -} - -/** - * Reciprocal Rank Fusion (RRF) algorithm for combining search results. - * - * RRF score = Σ (weight / (k + rank)) where rank is 1-indexed. - * - * @param bm25Results - Results from BM25 full-text search (ordered by relevance) - * @param semanticResults - Results from semantic/vector search (ordered by relevance) - * @param k - RRF constant (default 60, prevents high-ranked items from dominating) - * @param weights - Relative weights for each search type - */ -function rrfFusion( - bm25Results: Array<{ id: string }>, - semanticResults: Array<{ id: string }>, - k = 60, - weights = { fulltext: 1.0, semantic: 1.0 }, -): RankedResult[] { - const scores = new Map(); - - // Score from BM25 results - bm25Results.forEach((result, index) => { - const rank = index + 1; - const score = weights.fulltext / (k + rank); - scores.set(result.id, (scores.get(result.id) ?? 0) + score); - }); - - // Score from semantic results - semanticResults.forEach((result, index) => { - const rank = index + 1; - const score = weights.semantic / (k + rank); - scores.set(result.id, (scores.get(result.id) ?? 0) + score); - }); - - // Sort by combined RRF score (descending) - return Array.from(scores.entries()) - .map(([id, score]) => ({ id, score })) - .sort((a, b) => b.score - a.score); -} - -// ============================================================================= -// Temporal Parsing/Formatting -// ============================================================================= - -/** - * Parse a PostgreSQL tstzrange string into a temporal object - */ -function parseTemporal( - range: string | null, -): { start: Date; end: Date } | null { - if (!range) { - return null; - } - - // Parse format like ["2024-01-01 00:00:00+00","2024-01-02 00:00:00+00") - const match = range.match(/[[(]"?([^",]+)"?,"?([^",\])]+)"?[\])]/); - if (!match) { - return null; - } - - const startStr = match[1]; - const endStr = match[2]; - if (!startStr || !endStr) { - return null; - } - return { - start: new Date(startStr), - end: new Date(endStr), - }; -} - -/** - * Format a temporal object as a PostgreSQL tstzrange string - */ -function formatTemporal( - temporal: { start: Date; end?: Date } | null | undefined, -): string | null { - if (!temporal) { - return null; - } - - const start = temporal.start.toISOString(); - const end = temporal.end?.toISOString() ?? start; - - // Point-in-time: [same,same] (inclusive both ends) - // Range: [start,end) (inclusive-exclusive) - if (start === end) { - return `[${start},${end}]`; - } - return `[${start},${end})`; -} - -// ============================================================================= -// Row Conversion -// ============================================================================= - -function rowToMemory(row: MemoryRow): Memory { - return { - id: row.id, - content: row.content, - meta: row.meta, - tree: row.tree, - temporal: parseTemporal(row.temporal), - hasEmbedding: row.has_embedding, - createdAt: row.created_at, - createdBy: row.created_by, - updatedAt: row.updated_at, - }; -} - -function rowToSearchResult(row: SearchRow): SearchResultItem { - return { - ...rowToMemory(row), - score: typeof row.score === "string" ? parseFloat(row.score) : row.score, - }; -} - -// ============================================================================= -// Query Builders -// ============================================================================= - -interface FilterParams { - meta?: Record; - tree?: string; - temporal?: TemporalFilter; - grep?: string; -} - -/** - * Build common filter clauses for WHERE conditions - */ -function buildCommonFilters( - params: FilterParams, - valueOffset: number, -): { clauses: string[]; values: unknown[] } { - const clauses: string[] = []; - const values: unknown[] = []; - - // Metadata containment filter - if (params.meta && Object.keys(params.meta).length > 0) { - const paramIdx = valueOffset + values.length + 1; - clauses.push(`meta @> $${paramIdx}`); - values.push(params.meta); - } - - // Tree filter with auto-detection - if (params.tree) { - const paramIdx = valueOffset + values.length + 1; - const filterType = detectTreeFilterType(params.tree); - switch (filterType) { - case "ltxtquery": - clauses.push(`tree @ $${paramIdx}::ltxtquery`); - break; - case "lquery": - clauses.push(`tree ~ $${paramIdx}::lquery`); - break; - case "ltree": - clauses.push(`tree <@ $${paramIdx}::ltree`); - break; - } - values.push(params.tree); - } - - // Temporal filter - if (params.temporal) { - if (params.temporal.contains !== undefined) { - const paramIdx = valueOffset + values.length + 1; - clauses.push(`temporal @> $${paramIdx}::timestamptz`); - const ts = - params.temporal.contains instanceof Date - ? params.temporal.contains.toISOString() - : params.temporal.contains; - values.push(ts); - } else if (params.temporal.overlaps) { - const paramIdx1 = valueOffset + values.length + 1; - const paramIdx2 = valueOffset + values.length + 2; - clauses.push( - `temporal && tstzrange($${paramIdx1}::timestamptz, $${paramIdx2}::timestamptz, '[)')`, - ); - const [start, end] = params.temporal.overlaps; - values.push( - start instanceof Date ? start.toISOString() : start, - end instanceof Date ? end.toISOString() : end, - ); - } else if (params.temporal.within) { - const paramIdx1 = valueOffset + values.length + 1; - const paramIdx2 = valueOffset + values.length + 2; - clauses.push( - `temporal <@ tstzrange($${paramIdx1}::timestamptz, $${paramIdx2}::timestamptz, '[)')`, - ); - const [start, end] = params.temporal.within; - values.push( - start instanceof Date ? start.toISOString() : start, - end instanceof Date ? end.toISOString() : end, - ); - } - } - - // Content regex filter (POSIX, case-insensitive) - if (params.grep) { - const paramIdx = valueOffset + values.length + 1; - clauses.push(`content ~* $${paramIdx}`); - values.push(params.grep); - } - - return { clauses, values }; -} - -/** - * Build a BM25 full-text search query - */ -async function buildBM25Query( - sql: SQL, - schema: string, - params: FilterParams & { - query: string; - limit: number; - }, -): Promise { - const indexName = `${schema}.memory_content_bm25_idx`; - const { clauses, values } = buildCommonFilters(params, 2); // $1=query, $2=limit - - const whereClause = clauses.length > 0 ? `AND ${clauses.join(" AND ")}` : ""; - - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at, - -(content <@> to_bm25query($1, '${indexName}')) as score - FROM ${schema}.memory - WHERE content <@> to_bm25query($1, '${indexName}') < 0 - ${whereClause} - ORDER BY score DESC, created_at DESC - LIMIT $2 - `; - - return sql.unsafe(query, [ - params.query, - params.limit, - ...values, - ]); -} - -/** - * Build a semantic/vector similarity search query - */ -async function buildSemanticQuery( - sql: SQL, - schema: string, - params: FilterParams & { - embedding: number[]; - limit: number; - semanticThreshold?: number; - }, -): Promise { - const hasSemanticThreshold = typeof params.semanticThreshold === "number"; - const { clauses, values } = buildCommonFilters( - params, - hasSemanticThreshold ? 3 : 2, - ); // $1=embedding, $2=limit, optional $3=semanticThreshold - - const semanticThresholdClause = hasSemanticThreshold - ? "AND (1 - (embedding <=> $1::halfvec)) >= $3" - : "AND (embedding <=> $1::halfvec) < 1.0"; - const whereClause = clauses.length > 0 ? `AND ${clauses.join(" AND ")}` : ""; - - // Format embedding as PostgreSQL array literal - const embeddingLiteral = `[${params.embedding.join(",")}]`; - - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at, - (1 - (embedding <=> $1::halfvec)) as score - FROM ${schema}.memory - WHERE embedding IS NOT NULL - ${semanticThresholdClause} - ${whereClause} - ORDER BY score DESC, created_at DESC - LIMIT $2 - `; - - return sql.unsafe(query, [ - embeddingLiteral, - params.limit, - ...(hasSemanticThreshold ? [params.semanticThreshold] : []), - ...values, - ]); -} - -/** - * Build a filter-only query (no search ranking) - */ -async function buildFilterQuery( - sql: SQL, - schema: string, - params: FilterParams & { - limit: number; - orderBy: "asc" | "desc"; - }, -): Promise { - const { clauses, values } = buildCommonFilters(params, 1); // $1=limit - - const whereClause = - clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""; - const orderDirection = params.orderBy === "asc" ? "ASC" : "DESC"; - - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at, - 1.0 as score - FROM ${schema}.memory - ${whereClause} - ORDER BY created_at ${orderDirection} - LIMIT $1 - `; - - return sql.unsafe(query, [params.limit, ...values]); -} - -/** - * Fetch full memory rows by IDs, preserving order - */ -async function fetchByIds( - sql: SQL, - schema: string, - ids: string[], -): Promise { - if (ids.length === 0) { - return []; - } - - // Use array position to preserve order - const idsArray = `{${ids.join(",")}}`; - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at - FROM ${schema}.memory - WHERE id = ANY($1::uuid[]) - ORDER BY array_position($1::uuid[], id) - `; - - return sql.unsafe(query, [idsArray]); -} - -// ============================================================================= -// Memory Ops -// ============================================================================= - -export function memoryOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Create a new memory - */ - async createMemory(params: CreateMemoryParams): Promise { - const { id, content, meta = {}, tree = "", temporal, createdBy } = params; - - const temporalStr = formatTemporal(temporal); - - return withTx(ctx, "write", "createMemory", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.memory - (${id ? sql`id,` : sql``} content, meta, tree, temporal, created_by) - values - (${id ? sql`${id},` : sql``} ${content}, ${meta}::jsonb, ${tree}::ltree, ${temporalStr}::tstzrange, ${createdBy ?? null}) - returning - id, content, meta, tree::text, temporal::text, - embedding is not null as has_embedding, - created_at, created_by, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create memory"); - } - return rowToMemory(row); - }); - }, - - /** - * Batch create memories. - * - * Inserts skip duplicates by id (`ON CONFLICT (id) DO NOTHING`) so that - * a duplicate id in a single batch — or a retry of a partially-applied - * batch — does not abort the surrounding transaction. The returned ids - * may therefore be shorter than `params` when conflicts occur. Callers - * that care which inputs landed should compare ids against the input. - * - * This is safe because callers typically supply deterministic ids (e.g. - * the importer derives UUIDv7 from a stable hash of the source message), - * so a conflict means "we already have this row" rather than "two - * unrelated callers raced on a generated id." Callers without - * deterministic ids let the column default produce fresh UUIDv7s and - * never collide. - */ - async batchCreateMemories(params: CreateMemoryParams[]): Promise { - if (params.length === 0) { - return []; - } - - return withTx(ctx, "write", "batchCreateMemories", async (sql) => { - // TODO: Optimize with multi-row VALUES when Bun.sql supports it better - const ids: string[] = []; - for (const p of params) { - const temporalStr = formatTemporal(p.temporal); - const rows = await sql<{ id: string }[]>` - insert into ${sql.unsafe(schema)}.memory - (${p.id ? sql`id,` : sql``} content, meta, tree, temporal, created_by) - values - (${p.id ? sql`${p.id},` : sql``} ${p.content}, ${p.meta ?? {}}::jsonb, ${p.tree ?? ""}::ltree, ${temporalStr}::tstzrange, ${p.createdBy ?? null}) - on conflict (id) do nothing - returning id - `; - const row = rows[0]; - if (row) { - ids.push(row.id); - } - } - return ids; - }); - }, - - /** - * Get a memory by ID - */ - async getMemory(id: string): Promise { - return withTx(ctx, "read", "getMemory", async (sql) => { - const rows = await sql` - select - id, content, meta, tree::text, temporal::text, - embedding is not null as has_embedding, - created_at, created_by, updated_at - from ${sql.unsafe(schema)}.memory - where id = ${id} - `; - const row = rows[0]; - return row ? rowToMemory(row) : null; - }); - }, - - /** - * Update a memory - */ - async updateMemory( - id: string, - params: UpdateMemoryParams, - ): Promise { - const { content, meta, tree, temporal } = params; - - const updates: string[] = []; - const values: unknown[] = []; - let paramIndex = 1; - - if (content !== undefined) { - updates.push(`content = $${paramIndex++}`); - values.push(content); - } - if (meta !== undefined) { - updates.push(`meta = $${paramIndex++}::jsonb`); - values.push(meta); - } - if (tree !== undefined) { - updates.push(`tree = $${paramIndex++}::ltree`); - values.push(tree); - } - if (temporal !== undefined) { - updates.push(`temporal = $${paramIndex++}::tstzrange`); - values.push(formatTemporal(temporal)); - } - - if (updates.length === 0) { - return this.getMemory(id); - } - - values.push(id); - - return withTx(ctx, "write", "updateMemory", async (sql) => { - const query = ` - update ${schema}.memory - set ${updates.join(", ")} - where id = $${paramIndex} - returning - id, content, meta, tree::text, temporal::text, - embedding is not null as has_embedding, - created_at, created_by, updated_at - `; - - const rows = await sql.unsafe(query, values); - const row = rows[0]; - return row ? rowToMemory(row) : null; - }); - }, - - /** - * Delete a memory by ID - */ - async deleteMemory(id: string): Promise { - return withTx(ctx, "write", "deleteMemory", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.memory - where id = ${id} - `; - return result.count > 0; - }); - }, - - /** - * Delete all memories under a tree path - */ - async deleteTree(treePath: string): Promise<{ count: number }> { - return withTx(ctx, "write", "deleteTree", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.memory - where tree <@ ${treePath}::ltree - `; - return { count: result.count }; - }); - }, - - /** - * Count memories under a tree path (unbounded). - * - * Used to preview the impact of `deleteTree` / `moveTree` without - * being limited by a search `limit`. - */ - async countTree(treePath: string): Promise<{ count: number }> { - return withTx(ctx, "read", "countTree", async (sql) => { - const rows = await sql` - select count(*)::int as count - from ${sql.unsafe(schema)}.memory - where tree <@ ${treePath}::ltree - `; - return { count: Number(rows[0]?.count ?? 0) }; - }); - }, - - /** - * Move memories from one tree path to another - */ - async moveTree( - source: string, - destination: string, - ): Promise<{ count: number }> { - return withTx(ctx, "write", "moveTree", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.memory - set tree = case - when tree = ${source}::ltree then ${destination}::ltree - else ${destination}::ltree || subpath(tree, nlevel(${source}::ltree)) - end - where tree <@ ${source}::ltree - `; - return { count: result.count }; - }); - }, - - /** - * Search memories with hybrid BM25 + semantic search and RRF fusion. - * - * Search modes: - * 1. Hybrid (fulltext + embedding): Both searches run in parallel, results fused with RRF - * 2. BM25-only (fulltext): Full-text search using pg_textsearch - * 3. Semantic-only (embedding): Vector similarity search using pgvector - * 4. Filter-only (no search): Just filters by meta/tree/temporal - * - * Note: If `semantic` text is provided but no `embedding`, the caller is responsible - * for generating the embedding first. This keeps the embedding provider decoupled. - */ - async searchMemories(params: SearchParams): Promise { - const { - fulltext, - embedding, - grep, - meta, - tree, - temporal, - limit = 10, - candidateLimit = 30, - semanticThreshold, - weights = { fulltext: 1.0, semantic: 1.0 }, - orderBy = "desc", - } = params; - - return withTx(ctx, "read", "searchMemories", async (sql) => { - let results: SearchResultItem[]; - - if (fulltext && embedding && embedding.length > 0) { - // Case 1: Hybrid search with RRF fusion - const [bm25Results, semanticResults] = await Promise.all([ - buildBM25Query(sql, schema, { - query: fulltext, - grep, - meta, - tree, - temporal, - limit: candidateLimit, - }), - buildSemanticQuery(sql, schema, { - embedding, - grep, - meta, - tree, - temporal, - limit: candidateLimit, - semanticThreshold, - }), - ]); - - // Fuse results using RRF - const fusedResults = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: weights.fulltext ?? 1.0, - semantic: weights.semantic ?? 1.0, - }); - - // Take top N and fetch full records - const topIds = fusedResults.slice(0, limit).map((r) => r.id); - const scoreMap = new Map(fusedResults.map((r) => [r.id, r.score])); - - const rows = await fetchByIds(sql, schema, topIds); - results = rows.map((row) => ({ - ...rowToMemory(row), - score: scoreMap.get(row.id) ?? 0, - })); - } else if (fulltext) { - // Case 2: BM25-only search - const rows = await buildBM25Query(sql, schema, { - query: fulltext, - grep, - meta, - tree, - temporal, - limit, - }); - results = rows.map(rowToSearchResult); - } else if (embedding && embedding.length > 0) { - // Case 3: Semantic-only search - const rows = await buildSemanticQuery(sql, schema, { - embedding, - grep, - meta, - tree, - temporal, - limit, - semanticThreshold, - }); - results = rows.map(rowToSearchResult); - } else { - // Case 4: Filter-only (no search ranking) - const rows = await buildFilterQuery(sql, schema, { - grep, - meta, - tree, - temporal, - limit, - orderBy, - }); - results = rows.map(rowToSearchResult); - } - - return { - results, - total: results.length, - limit, - }; - }); - }, - - /** - * Get the tree structure with counts - */ - async getTree(params?: GetTreeParams): Promise { - const { tree: rootPath, levels } = params ?? {}; - - return withTx(ctx, "read", "getTree", async (sql) => { - if (rootPath) { - const rows = await sql` - select subpath(tree, 0, nlevel(${rootPath}::ltree) + g.lvl)::text as path, count(*)::int as count - from ${sql.unsafe(schema)}.memory - cross join lateral generate_series(1, ${levels ?? 100}) as g(lvl) - where tree <@ ${rootPath}::ltree - and nlevel(tree) >= nlevel(${rootPath}::ltree) + g.lvl - group by 1 - order by 1 - `; - return rows.map((r) => ({ path: r.path, count: r.count })); - } - - const rows = await sql` - select subpath(tree, 0, g.lvl)::text as path, count(*)::int as count - from ${sql.unsafe(schema)}.memory - cross join lateral generate_series(1, ${levels ?? 100}) as g(lvl) - where nlevel(tree) >= g.lvl - and tree <> ''::ltree - group by 1 - order by 1 - `; - return rows.map((r) => ({ path: r.path, count: r.count })); - }); - }, - }; -} - -export type MemoryOps = ReturnType; - -// Export for testing -export { detectTreeFilterType, rrfFusion }; diff --git a/packages/engine/ops/owner.ts b/packages/engine/ops/owner.ts deleted file mode 100644 index bc700e1..0000000 --- a/packages/engine/ops/owner.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { OpsContext, TreeOwner } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface TreeOwnerRow { - tree_path: string; - user_id: string; - user_name: string; - created_by: string | null; - created_by_name: string | null; - created_at: Date; -} - -function rowToTreeOwner(row: TreeOwnerRow): TreeOwner { - return { - treePath: row.tree_path, - userId: row.user_id, - userName: row.user_name, - createdBy: row.created_by, - createdByName: row.created_by_name, - createdAt: row.created_at, - }; -} - -export function ownerOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Set tree owner (upserts on tree_path) - */ - async setTreeOwner( - userId: string, - treePath: string, - createdBy?: string, - ): Promise { - await withTx(ctx, "admin", "setTreeOwner", async (sql) => { - await sql` - insert into ${sql.unsafe(schema)}.tree_owner - (tree_path, user_id, created_by) - values - (${treePath}::ltree, ${userId}, ${createdBy ?? null}) - on conflict (tree_path) - do update set - user_id = excluded.user_id, - created_by = excluded.created_by - `; - }); - }, - - /** - * Remove tree owner - */ - async removeTreeOwner(treePath: string): Promise { - return withTx(ctx, "admin", "removeTreeOwner", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.tree_owner - where tree_path = ${treePath}::ltree - `; - return result.count > 0; - }); - }, - - /** - * Get tree owner by path - */ - async getTreeOwner(treePath: string): Promise { - return withTx(ctx, "admin", "getTreeOwner", async (sql) => { - const rows = await sql` - select o.tree_path::text, o.user_id, u.name as user_name, o.created_by, cb.name as created_by_name, o.created_at - from ${sql.unsafe(schema)}.tree_owner o - join ${sql.unsafe(schema)}."user" u on u.id = o.user_id - left join ${sql.unsafe(schema)}."user" cb on cb.id = o.created_by - where o.tree_path = ${treePath}::ltree - `; - const row = rows[0]; - return row ? rowToTreeOwner(row) : null; - }); - }, - - /** - * List tree owners, optionally filtered by user - */ - async listTreeOwners(userId?: string): Promise { - return withTx(ctx, "admin", "listTreeOwners", async (sql) => { - if (userId) { - const rows = await sql` - select o.tree_path::text, o.user_id, u.name as user_name, o.created_by, cb.name as created_by_name, o.created_at - from ${sql.unsafe(schema)}.tree_owner o - join ${sql.unsafe(schema)}."user" u on u.id = o.user_id - left join ${sql.unsafe(schema)}."user" cb on cb.id = o.created_by - where o.user_id = ${userId} - order by o.tree_path - `; - return rows.map(rowToTreeOwner); - } - - const rows = await sql` - select o.tree_path::text, o.user_id, u.name as user_name, o.created_by, cb.name as created_by_name, o.created_at - from ${sql.unsafe(schema)}.tree_owner o - join ${sql.unsafe(schema)}."user" u on u.id = o.user_id - left join ${sql.unsafe(schema)}."user" cb on cb.id = o.created_by - order by o.tree_path - `; - return rows.map(rowToTreeOwner); - }); - }, - - /** - * Check if a user owns a tree path (or any ancestor) - */ - async isOwnerOf(userId: string, treePath: string): Promise { - return withTx(ctx, "admin", "isOwnerOf", async (sql) => { - const rows = await sql<{ is_owner: boolean }[]>` - select exists ( - select 1 - from ${sql.unsafe(schema)}.tree_owner - where user_id = ${userId} - and ${treePath}::ltree <@ tree_path - ) as is_owner - `; - return rows[0]?.is_owner ?? false; - }); - }, - }; -} - -export type OwnerOps = ReturnType; diff --git a/packages/engine/ops/role.ts b/packages/engine/ops/role.ts deleted file mode 100644 index 3e59b70..0000000 --- a/packages/engine/ops/role.ts +++ /dev/null @@ -1,142 +0,0 @@ -import type { OpsContext, RoleInfo, RoleMember } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface RoleMemberRow { - role_id: string; - member_id: string; - member_name: string; - with_admin_option: boolean; - created_at: Date; -} - -interface RoleInfoRow { - id: string; - name: string; - with_admin_option: boolean; -} - -function rowToRoleMember(row: RoleMemberRow): RoleMember { - return { - roleId: row.role_id, - memberId: row.member_id, - memberName: row.member_name, - withAdminOption: row.with_admin_option, - createdAt: row.created_at, - }; -} - -function rowToRoleInfo(row: RoleInfoRow): RoleInfo { - return { - id: row.id, - name: row.name, - withAdminOption: row.with_admin_option, - }; -} - -export function roleOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Add a member to a role (with cycle detection) - */ - async addRoleMember( - roleId: string, - memberId: string, - withAdminOption = false, - ): Promise { - await withTx(ctx, "admin", "addRoleMember", async (sql) => { - // Check for cycles first - const cycleRows = await sql<{ would_cycle: boolean }[]>` - select ${sql.unsafe(schema)}.would_create_cycle( - ${roleId}::uuid, - ${memberId}::uuid - ) as would_cycle - `; - - if (cycleRows[0]?.would_cycle) { - throw new Error( - `Adding member ${memberId} to role ${roleId} would create a cycle`, - ); - } - - await sql` - insert into ${sql.unsafe(schema)}.role_membership - (role_id, member_id, with_admin_option) - values - (${roleId}, ${memberId}, ${withAdminOption}) - on conflict (role_id, member_id) - do update set - with_admin_option = excluded.with_admin_option - `; - }); - }, - - /** - * Remove a member from a role - */ - async removeRoleMember(roleId: string, memberId: string): Promise { - return withTx(ctx, "admin", "removeRoleMember", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.role_membership - where role_id = ${roleId} - and member_id = ${memberId} - `; - return result.count > 0; - }); - }, - - /** - * List members of a role - */ - async listRoleMembers(roleId: string): Promise { - return withTx(ctx, "admin", "listRoleMembers", async (sql) => { - const rows = await sql` - select rm.role_id, rm.member_id, u.name as member_name, rm.with_admin_option, rm.created_at - from ${sql.unsafe(schema)}.role_membership rm - join ${sql.unsafe(schema)}."user" u on u.id = rm.member_id - where rm.role_id = ${roleId} - order by rm.created_at - `; - return rows.map(rowToRoleMember); - }); - }, - - /** - * List roles that a user is a member of - */ - async listRolesForUser(userId: string): Promise { - return withTx(ctx, "admin", "listRolesForUser", async (sql) => { - const rows = await sql` - select u.id, u.name, rm.with_admin_option - from ${sql.unsafe(schema)}.role_membership rm - join ${sql.unsafe(schema)}."user" u on u.id = rm.role_id - where rm.member_id = ${userId} - order by u.name - `; - return rows.map(rowToRoleInfo); - }); - }, - - /** - * Check if a user has admin option on a role - */ - async hasAdminOption(userId: string, roleId: string): Promise { - return withTx(ctx, "admin", "hasAdminOption", async (sql) => { - const rows = await sql<{ has_admin: boolean }[]>` - select exists ( - select 1 - from ${sql.unsafe(schema)}.role_membership - where role_id = ${roleId} - and member_id = ${userId} - and with_admin_option = true - ) as has_admin - `; - return rows[0]?.has_admin ?? false; - }); - }, - }; -} - -export type RoleOps = ReturnType; diff --git a/packages/engine/ops/user.ts b/packages/engine/ops/user.ts deleted file mode 100644 index 8bebd05..0000000 --- a/packages/engine/ops/user.ts +++ /dev/null @@ -1,193 +0,0 @@ -import type { CreateUserParams, OpsContext, User } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface UserRow { - id: string; - name: string; - identity_id: string | null; - can_login: boolean; - superuser: boolean; - createrole: boolean; - created_at: Date; - updated_at: Date | null; -} - -function rowToUser(row: UserRow): User { - return { - id: row.id, - name: row.name, - identityId: row.identity_id, - canLogin: row.can_login, - superuser: row.superuser, - createrole: row.createrole, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function userOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Create a new user - */ - async createUser(params: CreateUserParams): Promise { - const { - id, - name, - identityId = null, - canLogin = true, - superuser = false, - createrole = false, - } = params; - - return withTx(ctx, "admin", "createUser", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}."user" - (id, name, identity_id, can_login, superuser, createrole) - values - (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${name}, ${identityId}, ${canLogin}, ${superuser}, ${createrole}) - returning id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create user"); - } - return rowToUser(row); - }); - }, - - /** - * Create a role (user with can_login = false) - */ - async createRole(name: string, identityId?: string | null): Promise { - return this.createUser({ - name, - identityId, - canLogin: false, - superuser: false, - }); - }, - - /** - * Create a superuser - */ - async createSuperuser( - name: string, - id?: string, - identityId?: string | null, - ): Promise { - return this.createUser({ - id, - name, - identityId, - canLogin: true, - superuser: true, - }); - }, - - /** - * Get a user by ID - */ - async getUser(id: string): Promise { - return withTx(ctx, "admin", "getUser", async (sql) => { - const [row] = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where id = ${id} - `; - return row ? rowToUser(row) : null; - }); - }, - - /** - * Get a user by name - */ - async getUserByName(name: string): Promise { - return withTx(ctx, "admin", "getUserByName", async (sql) => { - const [row] = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where name = ${name} - `; - return row ? rowToUser(row) : null; - }); - }, - - /** - * List all users (optionally filter by can_login) - */ - async listUsers(canLogin?: boolean): Promise { - return withTx(ctx, "admin", "listUsers", async (sql) => { - const rows = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - ${canLogin !== undefined ? sql`where can_login = ${canLogin}` : sql``} - order by created_at - `; - return rows.map(rowToUser); - }); - }, - - /** - * List users linked to a specific identity - */ - async listUsersByIdentity(identityId: string): Promise { - return withTx(ctx, "admin", "listUsersByIdentity", async (sql) => { - const rows = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where identity_id = ${identityId} - order by created_at - `; - return rows.map(rowToUser); - }); - }, - - /** - * Find a user by identity ID (returns first match or null) - */ - async getUserByIdentity(identityId: string): Promise { - return withTx(ctx, "admin", "getUserByIdentity", async (sql) => { - const [row] = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where identity_id = ${identityId} - limit 1 - `; - return row ? rowToUser(row) : null; - }); - }, - - /** - * Rename a user - */ - async renameUser(id: string, newName: string): Promise { - return withTx(ctx, "admin", "renameUser", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}."user" - set name = ${newName} - where id = ${id} - `; - return result.count > 0; - }); - }, - - /** - * Delete a user - */ - async deleteUser(id: string): Promise { - return withTx(ctx, "admin", "deleteUser", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}."user" - where id = ${id} - `; - return result.count > 0; - }); - }, - }; -} - -export type UserOps = ReturnType; diff --git a/packages/engine/types.ts b/packages/engine/types.ts deleted file mode 100644 index da3c59a..0000000 --- a/packages/engine/types.ts +++ /dev/null @@ -1,258 +0,0 @@ -import type { SQL } from "bun"; - -// ============================================================================= -// Errors -// ============================================================================= - -/** - * Thrown when a feature is not yet implemented - */ -export class NotImplementedError extends Error { - constructor(message: string) { - super(message); - this.name = "NotImplementedError"; - } -} - -// ============================================================================= -// Context -// ============================================================================= - -/** - * Context passed to all ops functions - */ -export interface OpsContext { - /** Database connection or transaction handle */ - sql: SQL; - /** Schema name (e.g., "me_abc123xyz789") */ - schema: string; - /** Shard number for pgDog routing (optional, future use) */ - shard?: number; - /** Whether we're inside a transaction (controls whether withTx opens a new one) */ - inTransaction: boolean; - /** Get the current user ID (for RLS context) */ - getUserId: () => string | null; -} - -// ============================================================================= -// User -// ============================================================================= - -/** - * User: thing that accesses memories within an engine. - * Can be linked to an identity (soft FK to accounts.identity) or standalone. - * If can_login = false, it's a role (grant container for RBAC). - */ -export interface User { - id: string; - name: string; - identityId: string | null; - canLogin: boolean; - superuser: boolean; - createrole: boolean; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateUserParams { - id?: string; - name: string; - identityId?: string | null; - canLogin?: boolean; - superuser?: boolean; - createrole?: boolean; -} - -// ============================================================================= -// API Key -// ============================================================================= - -/** - * API key for authenticating to an engine. - * Scoped to a user within this engine. - */ -export interface ApiKey { - id: string; - userId: string; - lookupId: string; - name: string; - expiresAt: Date | null; - createdAt: Date; - revokedAt: Date | null; -} - -export interface CreateApiKeyParams { - userId: string; - name: string; - expiresAt?: Date | null; -} - -export interface CreateApiKeyResult { - apiKey: ApiKey; - /** The full API key string (only returned on creation) */ - rawKey: string; -} - -export interface ValidateApiKeyResult { - valid: boolean; - userId?: string; - apiKeyId?: string; - error?: string; -} - -// ============================================================================= -// Tree Grant -// ============================================================================= - -export interface TreeGrant { - id: string; - userId: string; - userName: string; - treePath: string; - actions: string[]; - grantedBy: string | null; - withGrantOption: boolean; - createdAt: Date; -} - -export interface GrantTreeAccessParams { - userId: string; - treePath: string; - actions: string[]; - grantedBy?: string | null; - withGrantOption?: boolean; -} - -// ============================================================================= -// Tree Owner -// ============================================================================= - -export interface TreeOwner { - treePath: string; - userId: string; - userName: string; - createdBy: string | null; - createdByName: string | null; - createdAt: Date; -} - -// ============================================================================= -// Role -// ============================================================================= - -export interface RoleMember { - roleId: string; - memberId: string; - memberName: string; - withAdminOption: boolean; - createdAt: Date; -} - -export interface RoleInfo { - id: string; - name: string; - withAdminOption: boolean; -} - -// ============================================================================= -// Memory -// ============================================================================= - -export interface Memory { - id: string; - content: string; - meta: Record; - tree: string; - temporal: { start: Date; end: Date } | null; - hasEmbedding: boolean; - createdAt: Date; - createdBy: string | null; - updatedAt: Date | null; -} - -export interface CreateMemoryParams { - id?: string; - content: string; - meta?: Record; - tree?: string; - temporal?: { start: Date; end?: Date } | null; - createdBy?: string | null; -} - -export interface UpdateMemoryParams { - content?: string; - meta?: Record; - tree?: string; - temporal?: { start: Date; end?: Date } | null; -} - -// ============================================================================= -// Search -// ============================================================================= - -export interface SearchParams { - /** Semantic search query (text - embedding must be generated by caller) */ - semantic?: string; - /** Pre-computed embedding vector for semantic search */ - embedding?: number[]; - /** Full-text (BM25) search query */ - fulltext?: string; - /** Regex filter applied to content (POSIX, case-insensitive) */ - grep?: string; - /** Filter by tree path (ltree, lquery, or ltxtquery) */ - tree?: string; - /** Filter by metadata (JSONB containment) */ - meta?: Record; - /** Temporal filter */ - temporal?: TemporalFilter; - /** Maximum results (default: 10) */ - limit?: number; - /** Candidates per search mode before RRF fusion */ - candidateLimit?: number; - /** Minimum semantic similarity score (0-1) for vector candidates */ - semanticThreshold?: number; - /** Weights for hybrid search */ - weights?: SearchWeights; - /** Sort direction for filter-only searches */ - orderBy?: "asc" | "desc"; -} - -export interface TemporalFilter { - /** Find memories containing this point in time */ - contains?: Date | string; - /** Find memories overlapping this range [start, end] */ - overlaps?: [Date | string, Date | string]; - /** Find memories fully within this range [start, end] */ - within?: [Date | string, Date | string]; -} - -export interface SearchWeights { - semantic?: number; - fulltext?: number; -} - -export interface SearchResult { - results: SearchResultItem[]; - total: number; - limit: number; -} - -export interface SearchResultItem extends Memory { - score: number; -} - -// ============================================================================= -// Tree -// ============================================================================= - -export interface GetTreeParams { - /** Root path to start from (default: root) */ - tree?: string; - /** Maximum depth to return */ - levels?: number; -} - -export interface TreeNode { - path: string; - count: number; -} diff --git a/packages/engine/util/api-key.test.ts b/packages/engine/util/api-key.test.ts deleted file mode 100644 index 3bc1942..0000000 --- a/packages/engine/util/api-key.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { - extractEngineSlug, - formatApiKey, - generateLookupId, - generateSecret, - hashSecret, - parseApiKey, - verifySecret, -} from "./api-key"; - -describe("generateLookupId", () => { - test("generates 16-char string", () => { - const id = generateLookupId(); - expect(id).toHaveLength(16); - }); - - test("only contains valid characters", () => { - const id = generateLookupId(); - expect(id).toMatch(/^[A-Za-z0-9_-]{16}$/); - }); - - test("generates unique values", () => { - const ids = new Set(Array.from({ length: 100 }, () => generateLookupId())); - expect(ids.size).toBe(100); - }); -}); - -describe("generateSecret", () => { - test("generates 32-char string", () => { - const secret = generateSecret(); - expect(secret).toHaveLength(32); - }); - - test("only contains base64url characters", () => { - const secret = generateSecret(); - expect(secret).toMatch(/^[A-Za-z0-9_-]{32}$/); - }); - - test("generates unique values", () => { - const secrets = new Set( - Array.from({ length: 100 }, () => generateSecret()), - ); - expect(secrets.size).toBe(100); - }); -}); - -describe("hashSecret / verifySecret", () => { - test("hash and verify round-trip", async () => { - const secret = generateSecret(); - const hash = await hashSecret(secret); - - expect(await verifySecret(secret, hash)).toBe(true); - expect(await verifySecret("wrong-secret", hash)).toBe(false); - }); - - test("different secrets produce different hashes", async () => { - const secret1 = generateSecret(); - const secret2 = generateSecret(); - const hash1 = await hashSecret(secret1); - const hash2 = await hashSecret(secret2); - - expect(hash1).not.toBe(hash2); - }); -}); - -describe("formatApiKey", () => { - test("formats key with all parts", () => { - const key = formatApiKey( - "abc123xyz789", - "Sh00uLs5rmSHHun3", - "secret32charslong_______________", - ); - expect(key).toBe( - "me.abc123xyz789.Sh00uLs5rmSHHun3.secret32charslong_______________", - ); - }); -}); - -describe("parseApiKey", () => { - // 32-char secret for tests - const validSecret = "pREy3xfnbCpgUXiaBcDeFgHiJkLm1234"; - - test("parses valid key", () => { - const key = `me.abc123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - const parsed = parseApiKey(key); - - expect(parsed).toEqual({ - engineSlug: "abc123xyz789", - lookupId: "Sh00uLs5rmSHHun3", - secret: validSecret, - }); - }); - - test("returns null for wrong prefix", () => { - const key = `xx.abc123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for invalid engineSlug (uppercase)", () => { - const key = `me.ABC123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for short engineSlug", () => { - const key = `me.abc123.Sh00uLs5rmSHHun3.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for invalid lookupId", () => { - const key = `me.abc123xyz789.short.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for wrong secret length", () => { - const key = "me.abc123xyz789.Sh00uLs5rmSHHun3.tooshort"; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for wrong number of parts", () => { - expect(parseApiKey("me.abc123xyz789.Sh00uLs5rmSHHun3")).toBeNull(); - expect(parseApiKey("me.abc123xyz789")).toBeNull(); - expect(parseApiKey("invalid")).toBeNull(); - }); -}); - -describe("extractEngineSlug", () => { - const validSecret = "pREy3xfnbCpgUXiaBcDeFgHiJkLm1234"; - - test("extracts slug from valid key", () => { - const key = `me.abc123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - expect(extractEngineSlug(key)).toBe("abc123xyz789"); - }); - - test("returns null for invalid key", () => { - expect(extractEngineSlug("invalid")).toBeNull(); - expect(extractEngineSlug("me.INVALID.x.y")).toBeNull(); - }); -}); diff --git a/packages/engine/util/api-key.ts b/packages/engine/util/api-key.ts deleted file mode 100644 index 09b4f77..0000000 --- a/packages/engine/util/api-key.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * API key generation and parsing utilities - * - * Key format: me.{engineSlug}.{lookupId}.{secret} - * Example: me.k8xf2nq4mp7a.Sh00uLs5rmSHHun3.pREy3xfnbCpgUXiaBcD... - * - * - me: Fixed prefix for all memory engine keys - * - engineSlug: 12-char alphanumeric identifier for routing - * - lookupId: 16-char alphanumeric identifier for database lookup - * - secret: 32-char random secret, verified against hash - */ - -const LOOKUP_ID_LENGTH = 16; -const SECRET_LENGTH = 32; -const LOOKUP_ID_CHARSET = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-"; - -/** - * Generate a random lookup ID (16 chars, URL-safe) - */ -export function generateLookupId(): string { - const bytes = crypto.getRandomValues(new Uint8Array(LOOKUP_ID_LENGTH)); - let result = ""; - for (const byte of bytes) { - result += LOOKUP_ID_CHARSET[byte % LOOKUP_ID_CHARSET.length]; - } - return result; -} - -/** - * Generate a random secret (32 chars, base64url) - */ -export function generateSecret(): string { - const bytes = crypto.getRandomValues(new Uint8Array(SECRET_LENGTH)); - return btoa(String.fromCharCode(...bytes)) - .replace(/\+/g, "-") - .replace(/\//g, "_") - .replace(/=/g, "") - .slice(0, SECRET_LENGTH); -} - -/** - * Hash a secret for storage using Argon2id - */ -export async function hashSecret(secret: string): Promise { - return Bun.password.hash(secret, { - algorithm: "argon2id", - memoryCost: 19456, - timeCost: 2, - }); -} - -/** - * Verify a secret against its hash - */ -export async function verifySecret( - secret: string, - hash: string, -): Promise { - return Bun.password.verify(secret, hash); -} - -/** - * Format a complete API key from its parts - */ -export function formatApiKey( - engineSlug: string, - lookupId: string, - secret: string, -): string { - return `me.${engineSlug}.${lookupId}.${secret}`; -} - -/** - * Parse an API key into its components - * Returns null if the key format is invalid - */ -export function parseApiKey( - key: string, -): { engineSlug: string; lookupId: string; secret: string } | null { - const parts = key.split("."); - if (parts.length !== 4) { - return null; - } - - const [prefix, engineSlug, lookupId, secret] = parts; - - // Validate prefix - if (prefix !== "me") { - return null; - } - - // Validate engineSlug format (12 lowercase alphanumeric chars) - if (!engineSlug || !/^[a-z0-9]{12}$/.test(engineSlug)) { - return null; - } - - // Validate lookupId format (16 chars from our charset) - if (!lookupId || !/^[A-Za-z0-9_-]{16}$/.test(lookupId)) { - return null; - } - - // Validate secret (32 chars, base64url) - if (!secret || secret.length !== SECRET_LENGTH) { - return null; - } - - return { engineSlug, lookupId, secret }; -} - -/** - * Extract the engine slug from an API key without full parsing - * Useful for routing before validation - */ -export function extractEngineSlug(key: string): string | null { - const parts = key.split("."); - if (parts.length !== 4 || parts[0] !== "me") { - return null; - } - const slug = parts[1]; - if (!slug || !/^[a-z0-9]{12}$/.test(slug)) { - return null; - } - return slug; -} diff --git a/packages/engine/util/index.ts b/packages/engine/util/index.ts deleted file mode 100644 index ea06279..0000000 --- a/packages/engine/util/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { - extractEngineSlug, - formatApiKey, - generateLookupId, - generateSecret, - hashSecret, - parseApiKey, - verifySecret, -} from "./api-key"; diff --git a/packages/server/middleware/authenticate-space.integration.test.ts b/packages/server/middleware/authenticate-space.integration.test.ts index b5632a1..23a48f4 100644 --- a/packages/server/middleware/authenticate-space.integration.test.ts +++ b/packages/server/middleware/authenticate-space.integration.test.ts @@ -13,7 +13,7 @@ import { migrateAuth, migrateCore, } from "@memory.build/database"; -import { core as engineCore } from "@memory.build/engine"; +import * as engineCore from "@memory.build/engine/core"; import postgres, { type Sql } from "postgres"; import { provisionUser } from "../provision"; import { authenticateSpace, SPACE_HEADER } from "./authenticate-space"; diff --git a/packages/server/provision.integration.test.ts b/packages/server/provision.integration.test.ts index e65a1f1..b9f94ba 100644 --- a/packages/server/provision.integration.test.ts +++ b/packages/server/provision.integration.test.ts @@ -12,7 +12,7 @@ import { migrateAuth, migrateCore, } from "@memory.build/database"; -import { core as engineCore } from "@memory.build/engine"; +import * as engineCore from "@memory.build/engine/core"; import postgres, { type Sql } from "postgres"; import { provisionUser } from "./provision"; diff --git a/packages/server/provision.ts b/packages/server/provision.ts index b3443d4..4f802c2 100644 --- a/packages/server/provision.ts +++ b/packages/server/provision.ts @@ -1,6 +1,6 @@ import { authStore, type OAuthProvider } from "@memory.build/auth"; import { generateSlug, provisionSpace } from "@memory.build/database"; -import { core as engineCore } from "@memory.build/engine"; +import * as engineCore from "@memory.build/engine/core"; import type { Sql } from "postgres"; /** diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index eb103a8..73ec6a3 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -11,8 +11,9 @@ import { migrateAuth, migrateCore, } from "@memory.build/database"; -import { core as engineCore, space as engineSpace } from "@memory.build/engine"; import type { TreeAccess } from "@memory.build/engine/core"; +import * as engineCore from "@memory.build/engine/core"; +import * as engineSpace from "@memory.build/engine/space"; import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; import postgres, { type Sql } from "postgres"; import { provisionUser } from "../../provision"; diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index 07f5d4d..a0d4814 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -13,9 +13,10 @@ import { migrateAuth, migrateCore, } from "@memory.build/database"; -import { core as engineCore, space as engineSpace } from "@memory.build/engine"; import type { TreeAccess } from "@memory.build/engine/core"; +import * as engineCore from "@memory.build/engine/core"; import type { SpaceStore } from "@memory.build/engine/space"; +import * as engineSpace from "@memory.build/engine/space"; import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; import postgres, { type Sql } from "postgres"; import { provisionUser } from "../../provision"; From 1f70c05e9c092d37b2dfa8d054a8472b2366869d Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 21:32:26 +0200 Subject: [PATCH 063/156] feat(client): drop the legacy engine + accounts clients; web on memoryClient (5D) - packages/web: the serve UI client switches from createClient to createMemoryClient (same /rpc proxy; the proxy injects Authorization + X-Me-Space, so the browser client carries neither). All memory.* calls and type imports are unchanged. - delete client/engine.ts + client/accounts.ts; trim the client barrel to the memory + user + auth clients. The public type surface now re-exports the memory data-plane schemas from @memory.build/protocol/engine/memory (was the whole legacy protocol/engine barrel) plus the namespace types from memory.ts. No remaining consumers of createClient/createAccountsClient/EngineClient/ AccountsClient. (Pre-existing, unrelated MonacoMarkdownEditor typecheck errors in packages/web are untouched.) typecheck + lint + unit (665) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/client/accounts.ts | 276 --------------------------- packages/client/engine.ts | 328 --------------------------------- packages/client/index.ts | 50 ++--- packages/web/src/api/client.ts | 9 +- 4 files changed, 20 insertions(+), 643 deletions(-) delete mode 100644 packages/client/accounts.ts delete mode 100644 packages/client/engine.ts diff --git a/packages/client/accounts.ts b/packages/client/accounts.ts deleted file mode 100644 index eebb3d4..0000000 --- a/packages/client/accounts.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Accounts client — for managing organizations, engines, and invitations. - * - * Authenticated via session token (obtained from the device flow login). - * Typically used by the CLI, not by end-user applications. - * - * @example - * ```ts - * import { createAccountsClient } from "@memory.build/client"; - * - * const accounts = createAccountsClient({ sessionToken: "..." }); - * - * const identity = await accounts.me.get(); - * const orgs = await accounts.org.list(); - * ``` - */ -import type { - AccountsMethodName, - AccountsParams, - AccountsResult, - EngineCreateParams, - EngineDeleteParams, - EngineDeleteResult, - EngineGetParams, - EngineListParams, - EngineListResult, - EngineResponse, - EngineSetupAccessParams, - EngineSetupAccessResult, - EngineUpdateParams, - IdentityGetByEmailParams, - IdentityGetByEmailResult, - IdentityResponse, - InvitationAcceptParams, - InvitationAcceptResult, - InvitationCreateParams, - InvitationCreateResult, - InvitationListParams, - InvitationListResult, - InvitationRevokeParams, - InvitationRevokeResult, - OrgCreateParams, - OrgDeleteParams, - OrgDeleteResult, - OrgGetParams, - OrgListResult, - OrgMemberAddParams, - OrgMemberListParams, - OrgMemberListResult, - OrgMemberRemoveParams, - OrgMemberRemoveResult, - OrgMemberResponse, - OrgMemberUpdateRoleParams, - OrgMemberUpdateRoleResult, - OrgResponse, - OrgUpdateParams, - SessionRevokeResult, -} from "@memory.build/protocol/accounts"; -import { rpcCall, type TransportConfig } from "./transport.ts"; - -// ============================================================================= -// Options -// ============================================================================= - -/** - * Options for creating an accounts client. - */ -export interface AccountsClientOptions { - /** Base URL of the Memory Engine server (default: "https://api.memory.build") */ - url?: string; - /** Session token for authentication */ - sessionToken?: string; - /** Request timeout in milliseconds (default: 30000) */ - timeout?: number; - /** Maximum retry attempts for transient failures (default: 3) */ - retries?: number; - /** - * CLIENT_VERSION of the caller. When set, sent as the `X-Client-Version` - * header on every RPC so the server can reject too-old clients with a - * typed `CLIENT_VERSION_INCOMPATIBLE` error before dispatch. - */ - clientVersion?: string; -} - -// ============================================================================= -// Namespace Types -// ============================================================================= - -export interface MeNamespace { - get(): Promise; -} - -export interface IdentityNamespace { - getByEmail( - params: IdentityGetByEmailParams, - ): Promise; -} - -export interface SessionNamespace { - revoke(): Promise; -} - -export interface OrgNamespace { - create(params: OrgCreateParams): Promise; - list(): Promise; - get(params: OrgGetParams): Promise; - update(params: OrgUpdateParams): Promise; - delete(params: OrgDeleteParams): Promise; - member: OrgMemberNamespace; -} - -export interface OrgMemberNamespace { - list(params: OrgMemberListParams): Promise; - add(params: OrgMemberAddParams): Promise; - remove(params: OrgMemberRemoveParams): Promise; - updateRole( - params: OrgMemberUpdateRoleParams, - ): Promise; -} - -export interface AccountsEngineNamespace { - create(params: EngineCreateParams): Promise; - list(params: EngineListParams): Promise; - get(params: EngineGetParams): Promise; - update(params: EngineUpdateParams): Promise; - delete(params: EngineDeleteParams): Promise; - setupAccess( - params: EngineSetupAccessParams, - ): Promise; -} - -export interface InvitationNamespace { - create(params: InvitationCreateParams): Promise; - list(params: InvitationListParams): Promise; - revoke(params: InvitationRevokeParams): Promise; - accept(params: InvitationAcceptParams): Promise; -} - -// ============================================================================= -// Client Type -// ============================================================================= - -/** - * Accounts client. - */ -export interface AccountsClient { - /** Current identity */ - me: MeNamespace; - /** Identity lookup */ - identity: IdentityNamespace; - /** Session management */ - session: SessionNamespace; - /** Organization management */ - org: OrgNamespace; - /** Engine management */ - engine: AccountsEngineNamespace; - /** Invitation management */ - invitation: InvitationNamespace; - - /** - * Low-level typed RPC call. - * Prefer the namespace methods for convenience. - */ - call( - method: M, - params: AccountsParams, - ): Promise>; - - /** Update the session token at runtime. */ - setSessionToken(token: string): void; - /** Get the current session token. */ - getSessionToken(): string | undefined; -} - -// ============================================================================= -// Factory -// ============================================================================= - -const DEFAULT_URL = "https://api.memory.build"; -const ACCOUNTS_RPC_PATH = "/api/v1/accounts/rpc"; -const DEFAULT_TIMEOUT = 30_000; -const DEFAULT_RETRIES = 3; - -/** - * Create an accounts client. - * - * Used for managing organizations, engines, members, and invitations. - * Requires a session token obtained from the device flow login. - * - * @example - * ```ts - * const accounts = createAccountsClient({ sessionToken: "..." }); - * - * const identity = await accounts.me.get(); - * const { orgs } = await accounts.org.list(); - * ``` - */ -export function createAccountsClient( - options: AccountsClientOptions = {}, -): AccountsClient { - const config: TransportConfig = { - url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), - path: ACCOUNTS_RPC_PATH, - token: options.sessionToken, - timeout: options.timeout ?? DEFAULT_TIMEOUT, - retries: options.retries ?? DEFAULT_RETRIES, - clientVersion: options.clientVersion, - }; - - function call( - method: M, - params: AccountsParams, - ): Promise> { - return rpcCall>(config, method, params); - } - - const member: OrgMemberNamespace = { - list: (params) => call("org.member.list", params), - add: (params) => call("org.member.add", params), - remove: (params) => call("org.member.remove", params), - updateRole: (params) => call("org.member.updateRole", params), - }; - - const me: MeNamespace = { - get: () => call("me.get", {}), - }; - - const identity: IdentityNamespace = { - getByEmail: (params) => call("identity.getByEmail", params), - }; - - const session: SessionNamespace = { - revoke: () => call("session.revoke", {}), - }; - - const org: OrgNamespace = { - create: (params) => call("org.create", params), - list: () => call("org.list", {}), - get: (params) => call("org.get", params), - update: (params) => call("org.update", params), - delete: (params) => call("org.delete", params), - member, - }; - - const engine: AccountsEngineNamespace = { - create: (params) => call("engine.create", params), - list: (params) => call("engine.list", params), - get: (params) => call("engine.get", params), - update: (params) => call("engine.update", params), - delete: (params) => call("engine.delete", params), - setupAccess: (params) => call("engine.setupAccess", params), - }; - - const invitation: InvitationNamespace = { - create: (params) => call("invitation.create", params), - list: (params) => call("invitation.list", params), - revoke: (params) => call("invitation.revoke", params), - accept: (params) => call("invitation.accept", params), - }; - - return { - me, - identity, - session, - org, - engine, - invitation, - call, - setSessionToken(token: string) { - config.token = token; - }, - getSessionToken() { - return config.token; - }, - }; -} diff --git a/packages/client/engine.ts b/packages/client/engine.ts deleted file mode 100644 index fe3e9d5..0000000 --- a/packages/client/engine.ts +++ /dev/null @@ -1,328 +0,0 @@ -/** - * Engine client — the primary client for interacting with Memory Engine. - * - * Provides typed, namespaced access to all 34 engine RPC methods - * (memory, user, grant, owner, role, apiKey) authenticated via API key. - * - * @example - * ```ts - * import { createClient } from "@memory.build/client"; - * - * const me = createClient({ apiKey: "me.xxx.yyy" }); - * - * const memory = await me.memory.create({ content: "hello world" }); - * const results = await me.memory.search({ semantic: "hello" }); - * const tree = await me.memory.tree(); - * ``` - */ -import type { - ApiKeyCreateParams, - ApiKeyCreateResult, - ApiKeyDeleteParams, - ApiKeyDeleteResult, - ApiKeyGetParams, - ApiKeyListParams, - ApiKeyListResult, - ApiKeyResponse, - ApiKeyRevokeParams, - ApiKeyRevokeResult, - EngineMethodName, - EngineParams, - EngineResult, - GrantCheckParams, - GrantCheckResult, - GrantCreateParams, - GrantCreateResult, - GrantGetParams, - GrantListParams, - GrantListResult, - GrantResponse, - GrantRevokeParams, - GrantRevokeResult, - MemoryBatchCreateParams, - MemoryBatchCreateResult, - MemoryCountTreeParams, - MemoryCountTreeResult, - MemoryCreateParams, - MemoryDeleteParams, - MemoryDeleteResult, - MemoryDeleteTreeParams, - MemoryDeleteTreeResult, - MemoryGetParams, - MemoryMoveParams, - MemoryMoveResult, - MemoryResponse, - MemorySearchParams, - MemorySearchResult, - MemoryTreeParams, - MemoryTreeResult, - MemoryUpdateParams, - OwnerGetParams, - OwnerListParams, - OwnerListResult, - OwnerRemoveParams, - OwnerRemoveResult, - OwnerResponse, - OwnerSetParams, - OwnerSetResult, - RoleAddMemberParams, - RoleAddMemberResult, - RoleCreateParams, - RoleListForUserParams, - RoleListForUserResult, - RoleListMembersParams, - RoleListMembersResult, - RoleRemoveMemberParams, - RoleRemoveMemberResult, - RoleResponse, - UserCreateParams, - UserDeleteParams, - UserDeleteResult, - UserGetByNameParams, - UserGetParams, - UserListParams, - UserListResult, - UserRenameParams, - UserRenameResult, - UserResponse, -} from "@memory.build/protocol/engine"; -import { rpcCall, type TransportConfig } from "./transport.ts"; - -// ============================================================================= -// Options -// ============================================================================= - -/** - * Options for creating an engine client. - */ -export interface ClientOptions { - /** Base URL of the Memory Engine server (default: "https://api.memory.build") */ - url?: string; - /** Engine JSON-RPC endpoint path (default: "/api/v1/engine/rpc") */ - rpcPath?: string; - /** API key for authentication (format: "me.lookupId.secret") */ - apiKey?: string; - /** Request timeout in milliseconds (default: 30000) */ - timeout?: number; - /** Maximum retry attempts for transient failures (default: 3) */ - retries?: number; - /** - * CLIENT_VERSION of the caller. When set, sent as the `X-Client-Version` - * header on every RPC so the server can reject too-old clients with a - * typed `CLIENT_VERSION_INCOMPATIBLE` error before dispatch. - */ - clientVersion?: string; -} - -// ============================================================================= -// Namespace Types -// ============================================================================= - -export interface MemoryNamespace { - create(params: MemoryCreateParams): Promise; - batchCreate( - params: MemoryBatchCreateParams, - ): Promise; - get(params: MemoryGetParams): Promise; - update(params: MemoryUpdateParams): Promise; - delete(params: MemoryDeleteParams): Promise; - search(params: MemorySearchParams): Promise; - tree(params?: MemoryTreeParams): Promise; - move(params: MemoryMoveParams): Promise; - deleteTree(params: MemoryDeleteTreeParams): Promise; - countTree(params: MemoryCountTreeParams): Promise; -} - -export interface UserNamespace { - create(params: UserCreateParams): Promise; - get(params: UserGetParams): Promise; - getByName(params: UserGetByNameParams): Promise; - list(params?: UserListParams): Promise; - rename(params: UserRenameParams): Promise; - delete(params: UserDeleteParams): Promise; -} - -export interface GrantNamespace { - create(params: GrantCreateParams): Promise; - list(params?: GrantListParams): Promise; - get(params: GrantGetParams): Promise; - revoke(params: GrantRevokeParams): Promise; - check(params: GrantCheckParams): Promise; -} - -export interface RoleNamespace { - create(params: RoleCreateParams): Promise; - addMember(params: RoleAddMemberParams): Promise; - removeMember(params: RoleRemoveMemberParams): Promise; - listMembers(params: RoleListMembersParams): Promise; - listForUser(params: RoleListForUserParams): Promise; -} - -export interface OwnerNamespace { - set(params: OwnerSetParams): Promise; - get(params: OwnerGetParams): Promise; - remove(params: OwnerRemoveParams): Promise; - list(params?: OwnerListParams): Promise; -} - -export interface ApiKeyNamespace { - create(params: ApiKeyCreateParams): Promise; - get(params: ApiKeyGetParams): Promise; - list(params: ApiKeyListParams): Promise; - revoke(params: ApiKeyRevokeParams): Promise; - delete(params: ApiKeyDeleteParams): Promise; -} - -// ============================================================================= -// Client Type -// ============================================================================= - -/** - * Memory Engine client. - */ -export interface EngineClient { - /** Memory operations (create, search, tree, etc.) */ - memory: MemoryNamespace; - /** User management */ - user: UserNamespace; - /** Tree grant management */ - grant: GrantNamespace; - /** Role management */ - role: RoleNamespace; - /** Tree owner management */ - owner: OwnerNamespace; - /** API key management */ - apiKey: ApiKeyNamespace; - - /** - * Low-level typed RPC call. - * Prefer the namespace methods for convenience. - */ - call( - method: M, - params: EngineParams, - ): Promise>; - - /** Update the API key at runtime. */ - setApiKey(apiKey: string): void; - /** Get the current API key. */ - getApiKey(): string | undefined; -} - -// ============================================================================= -// Factory -// ============================================================================= - -const DEFAULT_URL = "https://api.memory.build"; -const ENGINE_RPC_PATH = "/api/v1/engine/rpc"; -const DEFAULT_TIMEOUT = 30_000; -const DEFAULT_RETRIES = 3; - -/** - * Create a Memory Engine client. - * - * This is the primary entry point for interacting with Memory Engine. - * It connects to the engine RPC endpoint using API key authentication. - * - * @example - * ```ts - * const me = createClient({ apiKey: "me.xxx.yyy" }); - * - * // Create a memory - * const memory = await me.memory.create({ - * content: "TypeScript was released in 2012", - * tree: "knowledge.programming", - * }); - * - * // Search memories - * const results = await me.memory.search({ - * semantic: "when was TypeScript created", - * }); - * ``` - */ -export function createClient(options: ClientOptions = {}): EngineClient { - const config: TransportConfig = { - url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), - path: options.rpcPath ?? ENGINE_RPC_PATH, - token: options.apiKey, - timeout: options.timeout ?? DEFAULT_TIMEOUT, - retries: options.retries ?? DEFAULT_RETRIES, - clientVersion: options.clientVersion, - }; - - function call( - method: M, - params: EngineParams, - ): Promise> { - return rpcCall>(config, method, params); - } - - const memory: MemoryNamespace = { - create: (params) => call("memory.create", params), - batchCreate: (params) => call("memory.batchCreate", params), - get: (params) => call("memory.get", params), - update: (params) => call("memory.update", params), - delete: (params) => call("memory.delete", params), - search: (params) => call("memory.search", params), - tree: (params) => call("memory.tree", params ?? {}), - move: (params) => call("memory.move", params), - deleteTree: (params) => call("memory.deleteTree", params), - countTree: (params) => call("memory.countTree", params), - }; - - const user: UserNamespace = { - create: (params) => call("user.create", params), - get: (params) => call("user.get", params), - getByName: (params) => call("user.getByName", params), - list: (params) => call("user.list", params ?? {}), - rename: (params) => call("user.rename", params), - delete: (params) => call("user.delete", params), - }; - - const grant: GrantNamespace = { - create: (params) => call("grant.create", params), - list: (params) => call("grant.list", params ?? {}), - get: (params) => call("grant.get", params), - revoke: (params) => call("grant.revoke", params), - check: (params) => call("grant.check", params), - }; - - const role: RoleNamespace = { - create: (params) => call("role.create", params), - addMember: (params) => call("role.addMember", params), - removeMember: (params) => call("role.removeMember", params), - listMembers: (params) => call("role.listMembers", params), - listForUser: (params) => call("role.listForUser", params), - }; - - const owner: OwnerNamespace = { - set: (params) => call("owner.set", params), - get: (params) => call("owner.get", params), - remove: (params) => call("owner.remove", params), - list: (params) => call("owner.list", params ?? {}), - }; - - const apiKey: ApiKeyNamespace = { - create: (params) => call("apiKey.create", params), - get: (params) => call("apiKey.get", params), - list: (params) => call("apiKey.list", params), - revoke: (params) => call("apiKey.revoke", params), - delete: (params) => call("apiKey.delete", params), - }; - - return { - memory, - user, - grant, - role, - owner, - apiKey, - call, - setApiKey(apiKey: string) { - config.token = apiKey; - }, - getApiKey() { - return config.token; - }, - }; -} diff --git a/packages/client/index.ts b/packages/client/index.ts index 56be121..34ab196 100644 --- a/packages/client/index.ts +++ b/packages/client/index.ts @@ -1,22 +1,23 @@ /** * @memory.build/client — Client library for Memory Engine. * - * Three clients for different use cases: + * Two clients, both authenticated by a bearer token: * - * - {@link createClient} — Engine client (API key auth). - * The primary client for memory operations, search, user/grant management. + * - {@link createMemoryClient} — space data-plane + management. + * Talks to /api/v1/memory/rpc with the active space carried as X-Me-Space. + * Memory CRUD/search plus principal/group/grant/apiKey management. * - * - {@link createAccountsClient} — Accounts client (session token auth). - * For managing organizations, engines, and invitations. Used by CLI. + * - {@link createUserClient} — session-only, user-scoped. + * Talks to /api/v1/user/rpc: whoami, agent lifecycle, space discovery. * - * - {@link createAuthClient} — Auth client (no auth). + * - {@link createAuthClient} — auth client (no auth). * OAuth device flow for CLI login. Returns a session token. * * @example * ```ts - * import { createClient } from "@memory.build/client"; + * import { createMemoryClient } from "@memory.build/client"; * - * const me = createClient({ apiKey: "me.xxx.yyy" }); + * const me = createMemoryClient({ token: sessionToken, space: "abc123def456" }); * * await me.memory.create({ * content: "TypeScript was released in 2012", @@ -29,7 +30,7 @@ * ``` */ -export type * from "@memory.build/protocol/engine"; +export type * from "@memory.build/protocol/engine/memory"; export type { Meta, SearchWeights, @@ -37,44 +38,23 @@ export type { TemporalFilter, } from "@memory.build/protocol/fields"; -export type { - AccountsClient, - AccountsClientOptions, - AccountsEngineNamespace, - InvitationNamespace, - MeNamespace, - OrgMemberNamespace, - OrgNamespace, - SessionNamespace, -} from "./accounts.ts"; -// Accounts client -export { createAccountsClient } from "./accounts.ts"; export type { AuthClient, AuthClientOptions, PollOptions } from "./auth.ts"; // Auth client export { createAuthClient, DeviceFlowError } from "./auth.ts"; -export type { - ApiKeyNamespace, - ClientOptions, - EngineClient, - GrantNamespace, - MemoryNamespace, - OwnerNamespace, - RoleNamespace, - UserNamespace, -} from "./engine.ts"; -// Engine client (legacy; removed in Phase 5) -export { createClient } from "./engine.ts"; // Errors export { isRpcError, RpcError } from "./errors.ts"; -// Memory client (new model: space data-plane + management) +// Memory client (space data-plane + management) export { + type ApiKeyNamespace, createMemoryClient, + type GrantNamespace, type GroupNamespace, type MemoryClient, type MemoryClientOptions, + type MemoryNamespace, type PrincipalNamespace, } from "./memory.ts"; -// User client (new model: agent lifecycle + space discovery/management) +// User client (session-only: whoami, agent lifecycle, space discovery) export { type AgentNamespace, createUserClient, diff --git a/packages/web/src/api/client.ts b/packages/web/src/api/client.ts index a99592d..b7895a4 100644 --- a/packages/web/src/api/client.ts +++ b/packages/web/src/api/client.ts @@ -2,12 +2,13 @@ * Shared Memory Engine client for the web UI. * * The browser talks to the same-origin `/rpc` proxy exposed by `me serve`. - * That proxy injects the stored API key, so this client intentionally has no - * API key configured. Vite proxies `/rpc` to `me serve` during local dev. + * That proxy injects the session token (Authorization) and the active space + * (X-Me-Space), so this client carries neither. Vite proxies `/rpc` to + * `me serve` during local dev. */ -import { createClient } from "@memory.build/client"; +import { createMemoryClient } from "@memory.build/client"; -export const memoryEngineClient = createClient({ +export const memoryEngineClient = createMemoryClient({ url: "", rpcPath: "/rpc", retries: 0, From 9141cf660fd15abae2277eeacd4f15ea7cb10a44 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 21:35:10 +0200 Subject: [PATCH 064/156] feat(protocol): relocate memory schemas to ./memory; delete engine + accounts (5E) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The memory data-plane schemas were the only non-legacy part of protocol/engine. Move engine/memory.ts -> memory.ts (@memory.build/protocol/memory) and delete the rest of protocol/engine/* (user/grant/owner/role/api-key + index) and all of protocol/accounts/* — both contracts are gone with their server endpoints. - repoint importers: server rpc/memory, client memory.ts + index.ts to /memory; cli chunk + importers from the engine root to /memory. - protocol/index barrel + package.json (exports, files, build:js) updated: drop ./engine + ./accounts, add ./memory. Test parity: the deleted engine grant/owner schema tests cover the legacy action-based grant model (read/create/update/delete + withGrantOption), which has no equivalent — the new 3-level grant.set is exercised by the rpc/memory management integration suite. typecheck + lint + unit (651) + integration (31) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/chunk.test.ts | 2 +- packages/cli/chunk.ts | 2 +- packages/cli/importers/index.ts | 2 +- packages/client/index.ts | 2 +- packages/client/memory.ts | 5 +- packages/protocol/accounts/engine.ts | 128 ------------ packages/protocol/accounts/identity.test.ts | 60 ------ packages/protocol/accounts/identity.ts | 50 ----- packages/protocol/accounts/index.ts | 162 --------------- packages/protocol/accounts/invitation.ts | 106 ---------- packages/protocol/accounts/org-member.test.ts | 43 ---- packages/protocol/accounts/org-member.ts | 99 --------- packages/protocol/accounts/org.ts | 88 -------- packages/protocol/accounts/session.ts | 29 --- packages/protocol/engine/api-key.ts | 112 ---------- packages/protocol/engine/grant.test.ts | 36 ---- packages/protocol/engine/grant.ts | 117 ----------- packages/protocol/engine/index.ts | 194 ------------------ packages/protocol/engine/owner.test.ts | 44 ---- packages/protocol/engine/owner.ts | 91 -------- packages/protocol/engine/role.ts | 136 ------------ packages/protocol/engine/user.ts | 116 ----------- packages/protocol/index.ts | 14 +- packages/protocol/{engine => }/memory.ts | 2 +- packages/protocol/package.json | 27 +-- packages/server/rpc/memory/memory.ts | 4 +- 26 files changed, 22 insertions(+), 1649 deletions(-) delete mode 100644 packages/protocol/accounts/engine.ts delete mode 100644 packages/protocol/accounts/identity.test.ts delete mode 100644 packages/protocol/accounts/identity.ts delete mode 100644 packages/protocol/accounts/index.ts delete mode 100644 packages/protocol/accounts/invitation.ts delete mode 100644 packages/protocol/accounts/org-member.test.ts delete mode 100644 packages/protocol/accounts/org-member.ts delete mode 100644 packages/protocol/accounts/org.ts delete mode 100644 packages/protocol/accounts/session.ts delete mode 100644 packages/protocol/engine/api-key.ts delete mode 100644 packages/protocol/engine/grant.test.ts delete mode 100644 packages/protocol/engine/grant.ts delete mode 100644 packages/protocol/engine/index.ts delete mode 100644 packages/protocol/engine/owner.test.ts delete mode 100644 packages/protocol/engine/owner.ts delete mode 100644 packages/protocol/engine/role.ts delete mode 100644 packages/protocol/engine/user.ts rename packages/protocol/{engine => }/memory.ts (99%) diff --git a/packages/cli/chunk.test.ts b/packages/cli/chunk.test.ts index fcfba94..5e62393 100644 --- a/packages/cli/chunk.test.ts +++ b/packages/cli/chunk.test.ts @@ -2,7 +2,7 @@ * Tests for the byte-aware chunker in `chunk.ts`. */ import { describe, expect, test } from "bun:test"; -import type { MemoryCreateParams } from "@memory.build/protocol/engine"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; import { approxMemoryBytes, type BatchCreateClient, diff --git a/packages/cli/chunk.ts b/packages/cli/chunk.ts index ac2a4c6..df7e05f 100644 --- a/packages/cli/chunk.ts +++ b/packages/cli/chunk.ts @@ -14,7 +14,7 @@ * that callers should reach for unless they need a custom budget. */ -import type { MemoryCreateParams } from "@memory.build/protocol/engine"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; /** * Hard cap on memories per `memory.batchCreate` call. Matches the protocol diff --git a/packages/cli/importers/index.ts b/packages/cli/importers/index.ts index 8c50aec..4257da1 100644 --- a/packages/cli/importers/index.ts +++ b/packages/cli/importers/index.ts @@ -14,7 +14,7 @@ * time and are expected to be rare. */ -import type { MemoryCreateParams } from "@memory.build/protocol/engine"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; import { batchCreateChunked } from "../chunk.ts"; import type { MemoryClient } from "../client.ts"; import type { ProgressReporter } from "./progress.ts"; diff --git a/packages/client/index.ts b/packages/client/index.ts index 34ab196..14fff9a 100644 --- a/packages/client/index.ts +++ b/packages/client/index.ts @@ -30,13 +30,13 @@ * ``` */ -export type * from "@memory.build/protocol/engine/memory"; export type { Meta, SearchWeights, Temporal, TemporalFilter, } from "@memory.build/protocol/fields"; +export type * from "@memory.build/protocol/memory"; export type { AuthClient, AuthClientOptions, PollOptions } from "./auth.ts"; // Auth client diff --git a/packages/client/memory.ts b/packages/client/memory.ts index 4faf185..08b447a 100644 --- a/packages/client/memory.ts +++ b/packages/client/memory.ts @@ -12,6 +12,8 @@ * await me.principal.list({}); * ``` */ + +import { SPACE_HEADER } from "@memory.build/protocol/headers"; import type { MemoryBatchCreateParams, MemoryBatchCreateResult, @@ -31,8 +33,7 @@ import type { MemoryTreeParams, MemoryTreeResult, MemoryUpdateParams, -} from "@memory.build/protocol/engine/memory"; -import { SPACE_HEADER } from "@memory.build/protocol/headers"; +} from "@memory.build/protocol/memory"; import type { ApiKeyCreateParams, ApiKeyCreateResult, diff --git a/packages/protocol/accounts/engine.ts b/packages/protocol/accounts/engine.ts deleted file mode 100644 index f4d5205..0000000 --- a/packages/protocol/accounts/engine.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Engine method schemas (accounts side) — params and results for engine.* RPC methods. - * - * These are the engine management methods on the accounts RPC endpoint, - * not to be confused with the engine's own RPC methods. - */ -import { z } from "zod"; -import { engineStatusSchema, nameSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * engine.create params. - */ -export const engineCreateParams = z.object({ - orgId: uuidv7Schema, - name: nameSchema, - language: z - .string() - .regex(/^[a-z_]+$/) - .optional() - .default("english"), -}); - -export type EngineCreateParams = z.infer; - -/** - * engine.list params. - */ -export const engineListParams = z.object({ - orgId: uuidv7Schema, -}); - -export type EngineListParams = z.infer; - -/** - * engine.get params. - */ -export const engineGetParams = z.object({ - id: uuidv7Schema, -}); - -export type EngineGetParams = z.infer; - -/** - * engine.update params. - */ -export const engineUpdateParams = z.object({ - id: uuidv7Schema, - name: nameSchema.optional(), - status: engineStatusSchema.optional(), -}); - -export type EngineUpdateParams = z.infer; - -/** - * engine.delete params. - */ -export const engineDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type EngineDeleteParams = z.infer; - -/** - * engine.delete result. - */ -export const engineDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type EngineDeleteResult = z.infer; - -/** - * engine.setupAccess params. - * Bootstraps engine access for a session-authenticated identity. - */ -export const engineSetupAccessParams = z.object({ - engineId: uuidv7Schema, - apiKeyName: z.string().min(1).optional(), -}); - -export type EngineSetupAccessParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single engine response — returned by create, get, update. - */ -export const engineResponse = z.object({ - id: z.string(), - orgId: z.string(), - slug: z.string(), - name: z.string(), - shardId: z.number(), - status: z.string(), - language: z.string(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type EngineResponse = z.infer; - -/** - * engine.list result. - */ -export const engineListResult = z.object({ - engines: z.array(engineResponse), -}); - -export type EngineListResult = z.infer; - -/** - * engine.setupAccess result. - */ -export const engineSetupAccessResult = z.object({ - rawKey: z.string(), - engineSlug: z.string(), - userId: z.string(), - engineName: z.string(), - orgName: z.string(), -}); - -export type EngineSetupAccessResult = z.infer; diff --git a/packages/protocol/accounts/identity.test.ts b/packages/protocol/accounts/identity.test.ts deleted file mode 100644 index a59e628..0000000 --- a/packages/protocol/accounts/identity.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Tests for identity protocol schemas. - */ -import { describe, expect, test } from "bun:test"; -import { - identityGetByEmailParams, - identityGetByEmailResult, - identityResponse, -} from "./identity.ts"; - -describe("identityGetByEmailParams", () => { - test("accepts valid email", () => { - expect( - identityGetByEmailParams.safeParse({ email: "a@b.com" }).success, - ).toBe(true); - }); - - test("rejects invalid email", () => { - expect(identityGetByEmailParams.safeParse({ email: "nope" }).success).toBe( - false, - ); - }); - - test("rejects missing email", () => { - expect(identityGetByEmailParams.safeParse({}).success).toBe(false); - }); -}); - -describe("identityGetByEmailResult", () => { - test("accepts identity with all fields", () => { - const result = identityGetByEmailResult.safeParse({ - identity: { - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "a@b.com", - name: "Alice", - createdAt: "2026-01-15T00:00:00.000Z", - updatedAt: null, - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts null identity", () => { - const result = identityGetByEmailResult.safeParse({ identity: null }); - expect(result.success).toBe(true); - }); -}); - -describe("identityResponse", () => { - test("accepts valid identity", () => { - const result = identityResponse.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "a@b.com", - name: "Alice", - createdAt: "2026-01-15T00:00:00.000Z", - updatedAt: null, - }); - expect(result.success).toBe(true); - }); -}); diff --git a/packages/protocol/accounts/identity.ts b/packages/protocol/accounts/identity.ts deleted file mode 100644 index a7e2874..0000000 --- a/packages/protocol/accounts/identity.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Identity method schemas — params and results for me.* RPC methods. - */ -import { z } from "zod"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * me.get params — no params needed, uses session identity. - */ -export const meGetParams = z.object({}); - -export type MeGetParams = z.infer; - -/** - * identity.getByEmail params. - */ -export const identityGetByEmailParams = z.object({ - email: z.string().email(), -}); - -export type IdentityGetByEmailParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Identity response — returned by me.get. - */ -export const identityResponse = z.object({ - id: z.string(), - email: z.string(), - name: z.string(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type IdentityResponse = z.infer; - -/** - * identity.getByEmail result — nullable (identity may not exist). - */ -export const identityGetByEmailResult = z.object({ - identity: identityResponse.nullable(), -}); - -export type IdentityGetByEmailResult = z.infer; diff --git a/packages/protocol/accounts/index.ts b/packages/protocol/accounts/index.ts deleted file mode 100644 index 509c4fd..0000000 --- a/packages/protocol/accounts/index.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Accounts RPC contract — maps method names to params/result schema pairs. - * - * Covers all 21 methods on POST /api/v1/accounts/rpc (session token auth). - */ -import type { z } from "zod"; - -// Domain schemas -import { - engineCreateParams, - engineDeleteParams, - engineDeleteResult, - engineGetParams, - engineListParams, - engineListResult, - engineResponse, - engineSetupAccessParams, - engineSetupAccessResult, - engineUpdateParams, -} from "./engine.ts"; -import { - identityGetByEmailParams, - identityGetByEmailResult, - identityResponse, - meGetParams, -} from "./identity.ts"; -import { - invitationAcceptParams, - invitationAcceptResult, - invitationCreateParams, - invitationCreateResult, - invitationListParams, - invitationListResult, - invitationRevokeParams, - invitationRevokeResult, -} from "./invitation.ts"; -import { - orgCreateParams, - orgDeleteParams, - orgDeleteResult, - orgGetParams, - orgListParams, - orgListResult, - orgResponse, - orgUpdateParams, -} from "./org.ts"; -import { - orgMemberAddParams, - orgMemberListParams, - orgMemberListResult, - orgMemberRemoveParams, - orgMemberRemoveResult, - orgMemberResponse, - orgMemberUpdateRoleParams, - orgMemberUpdateRoleResult, -} from "./org-member.ts"; -import { sessionRevokeParams, sessionRevokeResult } from "./session.ts"; - -export * from "./engine.ts"; -// Re-export all domain schemas -export * from "./identity.ts"; -export * from "./invitation.ts"; -export * from "./org.ts"; -export * from "./org-member.ts"; -export * from "./session.ts"; - -// ============================================================================= -// RPC Contract -// ============================================================================= - -/** - * Define a method with its params schema and result schema. - */ -function method( - params: TParams, - result: TResult, -) { - return { params, result }; -} - -/** - * Accounts RPC method contract — all 21 methods. - * - * Each entry maps a method name to its params and result Zod schemas. - * The client library uses this for type inference and optional response validation. - * The server uses the params schemas for input validation. - */ -export const accountsMethods = { - // Identity (2) - "me.get": method(meGetParams, identityResponse), - "identity.getByEmail": method( - identityGetByEmailParams, - identityGetByEmailResult, - ), - - // Session (1) - "session.revoke": method(sessionRevokeParams, sessionRevokeResult), - - // Org (5) - "org.create": method(orgCreateParams, orgResponse), - "org.list": method(orgListParams, orgListResult), - "org.get": method(orgGetParams, orgResponse), - "org.update": method(orgUpdateParams, orgResponse), - "org.delete": method(orgDeleteParams, orgDeleteResult), - - // Org Member (4) - "org.member.list": method(orgMemberListParams, orgMemberListResult), - "org.member.add": method(orgMemberAddParams, orgMemberResponse), - "org.member.remove": method(orgMemberRemoveParams, orgMemberRemoveResult), - "org.member.updateRole": method( - orgMemberUpdateRoleParams, - orgMemberUpdateRoleResult, - ), - - // Engine (6) - "engine.create": method(engineCreateParams, engineResponse), - "engine.list": method(engineListParams, engineListResult), - "engine.get": method(engineGetParams, engineResponse), - "engine.update": method(engineUpdateParams, engineResponse), - "engine.delete": method(engineDeleteParams, engineDeleteResult), - "engine.setupAccess": method( - engineSetupAccessParams, - engineSetupAccessResult, - ), - - // Invitation (4) - "invitation.create": method(invitationCreateParams, invitationCreateResult), - "invitation.list": method(invitationListParams, invitationListResult), - "invitation.revoke": method(invitationRevokeParams, invitationRevokeResult), - "invitation.accept": method(invitationAcceptParams, invitationAcceptResult), -} as const; - -// ============================================================================= -// Type Utilities -// ============================================================================= - -/** Union of all accounts method names. */ -export type AccountsMethodName = keyof typeof accountsMethods; - -/** Extract the params type for a given accounts method. */ -export type AccountsParams = z.infer< - (typeof accountsMethods)[M]["params"] ->; - -/** Extract the result type for a given accounts method. */ -export type AccountsResult = z.infer< - (typeof accountsMethods)[M]["result"] ->; - -/** Get the params schema for runtime validation. */ -export function getAccountsParamsSchema( - method: M, -) { - return accountsMethods[method].params; -} - -/** Get the result schema for runtime response validation. */ -export function getAccountsResultSchema( - method: M, -) { - return accountsMethods[method].result; -} diff --git a/packages/protocol/accounts/invitation.ts b/packages/protocol/accounts/invitation.ts deleted file mode 100644 index e82f8e1..0000000 --- a/packages/protocol/accounts/invitation.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Invitation method schemas — params and results for invitation.* RPC methods. - */ -import { z } from "zod"; -import { emailSchema, orgRoleSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * invitation.create params. - */ -export const invitationCreateParams = z.object({ - orgId: uuidv7Schema, - email: emailSchema, - role: orgRoleSchema, - expiresInDays: z.number().int().min(1).max(30).optional(), -}); - -export type InvitationCreateParams = z.infer; - -/** - * invitation.list params. - */ -export const invitationListParams = z.object({ - orgId: uuidv7Schema, -}); - -export type InvitationListParams = z.infer; - -/** - * invitation.revoke params. - */ -export const invitationRevokeParams = z.object({ - id: uuidv7Schema, -}); - -export type InvitationRevokeParams = z.infer; - -/** - * invitation.accept params. - * Token is the raw invitation token from the email link. - */ -export const invitationAcceptParams = z.object({ - token: z.string().min(1, "token is required"), -}); - -export type InvitationAcceptParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Invitation response — returned by list. - */ -export const invitationResponse = z.object({ - id: z.string(), - orgId: z.string(), - email: z.string(), - role: z.string(), - invitedBy: z.string(), - expiresAt: z.string(), - acceptedAt: z.string().nullable(), - createdAt: z.string(), -}); - -export type InvitationResponse = z.infer; - -/** - * invitation.create result — includes the raw token (only returned on creation). - */ -export const invitationCreateResult = invitationResponse.extend({ - token: z.string(), -}); - -export type InvitationCreateResult = z.infer; - -/** - * invitation.list result. - */ -export const invitationListResult = z.object({ - invitations: z.array(invitationResponse), -}); - -export type InvitationListResult = z.infer; - -/** - * invitation.revoke result. - */ -export const invitationRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type InvitationRevokeResult = z.infer; - -/** - * invitation.accept result. - */ -export const invitationAcceptResult = z.object({ - accepted: z.boolean(), - orgId: z.string(), -}); - -export type InvitationAcceptResult = z.infer; diff --git a/packages/protocol/accounts/org-member.test.ts b/packages/protocol/accounts/org-member.test.ts deleted file mode 100644 index f1b18aa..0000000 --- a/packages/protocol/accounts/org-member.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Tests for org-member protocol response schemas. - * - * Verifies the response schemas accept name and email fields. - */ -import { describe, expect, test } from "bun:test"; -import { orgMemberResponse } from "./org-member.ts"; - -describe("orgMemberResponse", () => { - test("accepts response with name and email", () => { - const result = orgMemberResponse.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "admin", - name: "Alice Smith", - email: "alice@example.com", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects response missing name", () => { - const result = orgMemberResponse.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "member", - email: "alice@example.com", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); - - test("rejects response missing email", () => { - const result = orgMemberResponse.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "member", - name: "Alice", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/protocol/accounts/org-member.ts b/packages/protocol/accounts/org-member.ts deleted file mode 100644 index 5a0444a..0000000 --- a/packages/protocol/accounts/org-member.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Org member method schemas — params and results for org.member.* RPC methods. - */ -import { z } from "zod"; -import { orgRoleSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * org.member.list params. - */ -export const orgMemberListParams = z.object({ - orgId: uuidv7Schema, -}); - -export type OrgMemberListParams = z.infer; - -/** - * org.member.add params. - */ -export const orgMemberAddParams = z.object({ - orgId: uuidv7Schema, - identityId: uuidv7Schema, - role: orgRoleSchema, -}); - -export type OrgMemberAddParams = z.infer; - -/** - * org.member.remove params. - */ -export const orgMemberRemoveParams = z.object({ - orgId: uuidv7Schema, - identityId: uuidv7Schema, -}); - -export type OrgMemberRemoveParams = z.infer; - -/** - * org.member.updateRole params. - */ -export const orgMemberUpdateRoleParams = z.object({ - orgId: uuidv7Schema, - identityId: uuidv7Schema, - role: orgRoleSchema, -}); - -export type OrgMemberUpdateRoleParams = z.infer< - typeof orgMemberUpdateRoleParams ->; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single org member response — returned by add, used in list. - */ -export const orgMemberResponse = z.object({ - orgId: z.string(), - identityId: z.string(), - role: z.string(), - name: z.string(), - email: z.string(), - createdAt: z.string(), -}); - -export type OrgMemberResponse = z.infer; - -/** - * org.member.list result. - */ -export const orgMemberListResult = z.object({ - members: z.array(orgMemberResponse), -}); - -export type OrgMemberListResult = z.infer; - -/** - * org.member.remove result. - */ -export const orgMemberRemoveResult = z.object({ - removed: z.boolean(), -}); - -export type OrgMemberRemoveResult = z.infer; - -/** - * org.member.updateRole result. - */ -export const orgMemberUpdateRoleResult = z.object({ - updated: z.boolean(), -}); - -export type OrgMemberUpdateRoleResult = z.infer< - typeof orgMemberUpdateRoleResult ->; diff --git a/packages/protocol/accounts/org.ts b/packages/protocol/accounts/org.ts deleted file mode 100644 index 5bba03b..0000000 --- a/packages/protocol/accounts/org.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Org method schemas — params and results for org.* RPC methods. - */ -import { z } from "zod"; -import { nameSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * org.create params. - */ -export const orgCreateParams = z.object({ - name: nameSchema, -}); - -export type OrgCreateParams = z.infer; - -/** - * org.list params — no params needed, lists orgs for session identity. - */ -export const orgListParams = z.object({}); - -export type OrgListParams = z.infer; - -/** - * org.get params. - */ -export const orgGetParams = z.object({ - id: uuidv7Schema, -}); - -export type OrgGetParams = z.infer; - -/** - * org.update params. - */ -export const orgUpdateParams = z.object({ - id: uuidv7Schema, - name: nameSchema.optional(), -}); - -export type OrgUpdateParams = z.infer; - -/** - * org.delete params. - */ -export const orgDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type OrgDeleteParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single org response — returned by create, get, update. - */ -export const orgResponse = z.object({ - id: z.string(), - slug: z.string(), - name: z.string(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type OrgResponse = z.infer; - -/** - * org.list result. - */ -export const orgListResult = z.object({ - orgs: z.array(orgResponse), -}); - -export type OrgListResult = z.infer; - -/** - * org.delete result. - */ -export const orgDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type OrgDeleteResult = z.infer; diff --git a/packages/protocol/accounts/session.ts b/packages/protocol/accounts/session.ts deleted file mode 100644 index 97ce82d..0000000 --- a/packages/protocol/accounts/session.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Session method schemas — params and results for session.* RPC methods. - */ -import { z } from "zod"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * session.revoke params — revokes the current session (logout). - * No params needed — uses the session from the auth token. - */ -export const sessionRevokeParams = z.object({}); - -export type SessionRevokeParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * session.revoke result. - */ -export const sessionRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type SessionRevokeResult = z.infer; diff --git a/packages/protocol/engine/api-key.ts b/packages/protocol/engine/api-key.ts deleted file mode 100644 index ba17a23..0000000 --- a/packages/protocol/engine/api-key.ts +++ /dev/null @@ -1,112 +0,0 @@ -/** - * API Key method schemas — params and results for apiKey.* RPC methods. - */ -import { z } from "zod"; -import { timestampSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * apiKey.create params. - */ -export const apiKeyCreateParams = z.object({ - userId: uuidv7Schema, - name: z.string().min(1, "name is required"), - expiresAt: timestampSchema.optional().nullable(), -}); - -export type ApiKeyCreateParams = z.infer; - -/** - * apiKey.get params. - */ -export const apiKeyGetParams = z.object({ - id: uuidv7Schema, -}); - -export type ApiKeyGetParams = z.infer; - -/** - * apiKey.list params. - */ -export const apiKeyListParams = z.object({ - userId: uuidv7Schema, -}); - -export type ApiKeyListParams = z.infer; - -/** - * apiKey.revoke params. - */ -export const apiKeyRevokeParams = z.object({ - id: uuidv7Schema, -}); - -export type ApiKeyRevokeParams = z.infer; - -/** - * apiKey.delete params. - */ -export const apiKeyDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type ApiKeyDeleteParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single API key response — returned by get, included in list and create. - */ -export const apiKeyResponse = z.object({ - id: z.string(), - userId: z.string(), - lookupId: z.string(), - name: z.string(), - expiresAt: z.string().nullable(), - createdAt: z.string(), - revokedAt: z.string().nullable(), -}); - -export type ApiKeyResponse = z.infer; - -/** - * apiKey.create result — includes the raw key (only returned on creation). - */ -export const apiKeyCreateResult = z.object({ - apiKey: apiKeyResponse, - rawKey: z.string(), -}); - -export type ApiKeyCreateResult = z.infer; - -/** - * apiKey.list result. - */ -export const apiKeyListResult = z.object({ - apiKeys: z.array(apiKeyResponse), -}); - -export type ApiKeyListResult = z.infer; - -/** - * apiKey.revoke result. - */ -export const apiKeyRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type ApiKeyRevokeResult = z.infer; - -/** - * apiKey.delete result. - */ -export const apiKeyDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type ApiKeyDeleteResult = z.infer; diff --git a/packages/protocol/engine/grant.test.ts b/packages/protocol/engine/grant.test.ts deleted file mode 100644 index 839dfa3..0000000 --- a/packages/protocol/engine/grant.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Tests for grant protocol response schemas. - * - * Verifies the response schemas accept the userName field added via JOINs. - */ -import { describe, expect, test } from "bun:test"; -import { grantResponse } from "./grant.ts"; - -describe("grantResponse", () => { - test("accepts response with userName", () => { - const result = grantResponse.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - userId: "019d694f-79f6-7595-8faf-b70b01c11f99", - userName: "alice", - treePath: "work.projects", - actions: ["read", "create"], - grantedBy: null, - withGrantOption: false, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects response missing userName", () => { - const result = grantResponse.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - userId: "019d694f-79f6-7595-8faf-b70b01c11f99", - treePath: "work.projects", - actions: ["read"], - grantedBy: null, - withGrantOption: false, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/protocol/engine/grant.ts b/packages/protocol/engine/grant.ts deleted file mode 100644 index 4973e4e..0000000 --- a/packages/protocol/engine/grant.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Grant method schemas — params and results for grant.* RPC methods. - */ -import { z } from "zod"; -import { grantActionSchema, treePathSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * grant.create params. - */ -export const grantCreateParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, - actions: z.array(grantActionSchema).min(1, "at least one action required"), - withGrantOption: z.boolean().optional(), -}); - -export type GrantCreateParams = z.infer; - -/** - * grant.list params. - */ -export const grantListParams = z.object({ - userId: uuidv7Schema.optional(), -}); - -export type GrantListParams = z.infer; - -/** - * grant.get params. - */ -export const grantGetParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, -}); - -export type GrantGetParams = z.infer; - -/** - * grant.revoke params. - */ -export const grantRevokeParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, -}); - -export type GrantRevokeParams = z.infer; - -/** - * grant.check params. - */ -export const grantCheckParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, - action: grantActionSchema, -}); - -export type GrantCheckParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single grant response — returned by get. - */ -export const grantResponse = z.object({ - id: z.string(), - userId: z.string(), - userName: z.string(), - treePath: z.string(), - actions: z.array(z.string()), - grantedBy: z.string().nullable(), - withGrantOption: z.boolean(), - createdAt: z.string(), -}); - -export type GrantResponse = z.infer; - -/** - * grant.create result. - */ -export const grantCreateResult = z.object({ - created: z.boolean(), -}); - -export type GrantCreateResult = z.infer; - -/** - * grant.list result. - */ -export const grantListResult = z.object({ - grants: z.array(grantResponse), -}); - -export type GrantListResult = z.infer; - -/** - * grant.revoke result. - */ -export const grantRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type GrantRevokeResult = z.infer; - -/** - * grant.check result. - */ -export const grantCheckResult = z.object({ - allowed: z.boolean(), -}); - -export type GrantCheckResult = z.infer; diff --git a/packages/protocol/engine/index.ts b/packages/protocol/engine/index.ts deleted file mode 100644 index c035bda..0000000 --- a/packages/protocol/engine/index.ts +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Engine RPC contract — maps method names to params/result schema pairs. - * - * Covers all 35 methods on POST /api/v1/engine/rpc (API key auth). - */ -import type { z } from "zod"; - -// Domain schemas -import { - apiKeyCreateParams, - apiKeyCreateResult, - apiKeyDeleteParams, - apiKeyDeleteResult, - apiKeyGetParams, - apiKeyListParams, - apiKeyListResult, - apiKeyResponse, - apiKeyRevokeParams, - apiKeyRevokeResult, -} from "./api-key.ts"; -import { - grantCheckParams, - grantCheckResult, - grantCreateParams, - grantCreateResult, - grantGetParams, - grantListParams, - grantListResult, - grantResponse, - grantRevokeParams, - grantRevokeResult, -} from "./grant.ts"; -import { - memoryBatchCreateParams, - memoryBatchCreateResult, - memoryCountTreeParams, - memoryCountTreeResult, - memoryCreateParams, - memoryDeleteParams, - memoryDeleteResult, - memoryDeleteTreeParams, - memoryDeleteTreeResult, - memoryGetParams, - memoryMoveParams, - memoryMoveResult, - memoryResponse, - memorySearchParams, - memorySearchResult, - memoryTreeParams, - memoryTreeResult, - memoryUpdateParams, -} from "./memory.ts"; -import { - ownerGetParams, - ownerListParams, - ownerListResult, - ownerRemoveParams, - ownerRemoveResult, - ownerResponse, - ownerSetParams, - ownerSetResult, -} from "./owner.ts"; -import { - roleAddMemberParams, - roleAddMemberResult, - roleCreateParams, - roleListForUserParams, - roleListForUserResult, - roleListMembersParams, - roleListMembersResult, - roleRemoveMemberParams, - roleRemoveMemberResult, - roleResponse, -} from "./role.ts"; -import { - userCreateParams, - userDeleteParams, - userDeleteResult, - userGetByNameParams, - userGetParams, - userListParams, - userListResult, - userRenameParams, - userRenameResult, - userResponse, -} from "./user.ts"; - -export * from "./api-key.ts"; -export * from "./grant.ts"; -// Re-export all domain schemas -export * from "./memory.ts"; -export * from "./owner.ts"; -export * from "./role.ts"; -export * from "./user.ts"; - -// ============================================================================= -// RPC Contract -// ============================================================================= - -/** - * Define a method with its params schema and result schema. - */ -function method( - params: TParams, - result: TResult, -) { - return { params, result }; -} - -/** - * Engine RPC method contract — all 35 methods. - * - * Each entry maps a method name to its params and result Zod schemas. - * The client library uses this for type inference and optional response validation. - * The server uses the params schemas for input validation. - */ -export const engineMethods = { - // Memory (10) - "memory.create": method(memoryCreateParams, memoryResponse), - "memory.batchCreate": method( - memoryBatchCreateParams, - memoryBatchCreateResult, - ), - "memory.get": method(memoryGetParams, memoryResponse), - "memory.update": method(memoryUpdateParams, memoryResponse), - "memory.delete": method(memoryDeleteParams, memoryDeleteResult), - "memory.search": method(memorySearchParams, memorySearchResult), - "memory.tree": method(memoryTreeParams, memoryTreeResult), - "memory.move": method(memoryMoveParams, memoryMoveResult), - "memory.deleteTree": method(memoryDeleteTreeParams, memoryDeleteTreeResult), - "memory.countTree": method(memoryCountTreeParams, memoryCountTreeResult), - - // User (6) - "user.create": method(userCreateParams, userResponse), - "user.get": method(userGetParams, userResponse), - "user.getByName": method(userGetByNameParams, userResponse), - "user.list": method(userListParams, userListResult), - "user.rename": method(userRenameParams, userRenameResult), - "user.delete": method(userDeleteParams, userDeleteResult), - - // Grant (5) - "grant.create": method(grantCreateParams, grantCreateResult), - "grant.list": method(grantListParams, grantListResult), - "grant.get": method(grantGetParams, grantResponse), - "grant.revoke": method(grantRevokeParams, grantRevokeResult), - "grant.check": method(grantCheckParams, grantCheckResult), - - // Role (5) - "role.create": method(roleCreateParams, roleResponse), - "role.addMember": method(roleAddMemberParams, roleAddMemberResult), - "role.removeMember": method(roleRemoveMemberParams, roleRemoveMemberResult), - "role.listMembers": method(roleListMembersParams, roleListMembersResult), - "role.listForUser": method(roleListForUserParams, roleListForUserResult), - - // Owner (4) - "owner.set": method(ownerSetParams, ownerSetResult), - "owner.get": method(ownerGetParams, ownerResponse), - "owner.remove": method(ownerRemoveParams, ownerRemoveResult), - "owner.list": method(ownerListParams, ownerListResult), - - // API Key (5) - "apiKey.create": method(apiKeyCreateParams, apiKeyCreateResult), - "apiKey.get": method(apiKeyGetParams, apiKeyResponse), - "apiKey.list": method(apiKeyListParams, apiKeyListResult), - "apiKey.revoke": method(apiKeyRevokeParams, apiKeyRevokeResult), - "apiKey.delete": method(apiKeyDeleteParams, apiKeyDeleteResult), -} as const; - -// ============================================================================= -// Type Utilities -// ============================================================================= - -/** Union of all engine method names. */ -export type EngineMethodName = keyof typeof engineMethods; - -/** Extract the params type for a given engine method. */ -export type EngineParams = z.infer< - (typeof engineMethods)[M]["params"] ->; - -/** Extract the result type for a given engine method. */ -export type EngineResult = z.infer< - (typeof engineMethods)[M]["result"] ->; - -/** Get the params schema for runtime validation. */ -export function getEngineParamsSchema(method: M) { - return engineMethods[method].params; -} - -/** Get the result schema for runtime response validation. */ -export function getEngineResultSchema(method: M) { - return engineMethods[method].result; -} diff --git a/packages/protocol/engine/owner.test.ts b/packages/protocol/engine/owner.test.ts deleted file mode 100644 index 661949e..0000000 --- a/packages/protocol/engine/owner.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Tests for owner protocol response schemas. - * - * Verifies the response schemas accept userName and createdByName. - */ -import { describe, expect, test } from "bun:test"; -import { ownerResponse } from "./owner.ts"; - -describe("ownerResponse", () => { - test("accepts response with userName and createdByName", () => { - const result = ownerResponse.safeParse({ - treePath: "work.projects", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: "019d694f-79f6-7595-8faf-b70b01c11f99", - createdByName: "admin", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("accepts null createdBy and createdByName", () => { - const result = ownerResponse.safeParse({ - treePath: "work", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: null, - createdByName: null, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects response missing userName", () => { - const result = ownerResponse.safeParse({ - treePath: "work", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - createdBy: null, - createdByName: null, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/protocol/engine/owner.ts b/packages/protocol/engine/owner.ts deleted file mode 100644 index be0da06..0000000 --- a/packages/protocol/engine/owner.ts +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Owner method schemas — params and results for owner.* RPC methods. - */ -import { z } from "zod"; -import { treePathSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * owner.set params. - */ -export const ownerSetParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, -}); - -export type OwnerSetParams = z.infer; - -/** - * owner.remove params. - */ -export const ownerRemoveParams = z.object({ - treePath: treePathSchema, -}); - -export type OwnerRemoveParams = z.infer; - -/** - * owner.get params. - */ -export const ownerGetParams = z.object({ - treePath: treePathSchema, -}); - -export type OwnerGetParams = z.infer; - -/** - * owner.list params. - */ -export const ownerListParams = z.object({ - userId: uuidv7Schema.optional(), -}); - -export type OwnerListParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single owner response. - */ -export const ownerResponse = z.object({ - treePath: z.string(), - userId: z.string(), - userName: z.string(), - createdBy: z.string().nullable(), - createdByName: z.string().nullable(), - createdAt: z.string(), -}); - -export type OwnerResponse = z.infer; - -/** - * owner.set result. - */ -export const ownerSetResult = z.object({ - set: z.boolean(), -}); - -export type OwnerSetResult = z.infer; - -/** - * owner.remove result. - */ -export const ownerRemoveResult = z.object({ - removed: z.boolean(), -}); - -export type OwnerRemoveResult = z.infer; - -/** - * owner.list result. - */ -export const ownerListResult = z.object({ - owners: z.array(ownerResponse), -}); - -export type OwnerListResult = z.infer; diff --git a/packages/protocol/engine/role.ts b/packages/protocol/engine/role.ts deleted file mode 100644 index 4fa3b0e..0000000 --- a/packages/protocol/engine/role.ts +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Role method schemas — params and results for role.* RPC methods. - */ -import { z } from "zod"; -import { uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * role.create params. - * Creates a user with canLogin=false (a role for grouping grants). - */ -export const roleCreateParams = z.object({ - name: z.string().min(1, "name is required"), - identityId: uuidv7Schema.optional().nullable(), -}); - -export type RoleCreateParams = z.infer; - -/** - * role.addMember params. - */ -export const roleAddMemberParams = z.object({ - roleId: uuidv7Schema, - memberId: uuidv7Schema, - withAdminOption: z.boolean().optional(), -}); - -export type RoleAddMemberParams = z.infer; - -/** - * role.removeMember params. - */ -export const roleRemoveMemberParams = z.object({ - roleId: uuidv7Schema, - memberId: uuidv7Schema, -}); - -export type RoleRemoveMemberParams = z.infer; - -/** - * role.listMembers params. - */ -export const roleListMembersParams = z.object({ - roleId: uuidv7Schema, -}); - -export type RoleListMembersParams = z.infer; - -/** - * role.listForUser params. - */ -export const roleListForUserParams = z.object({ - userId: uuidv7Schema, -}); - -export type RoleListForUserParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single role response — returned by create. - */ -export const roleResponse = z.object({ - id: z.string(), - name: z.string(), - identityId: z.string().nullable(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type RoleResponse = z.infer; - -/** - * Role member response — used in listMembers result. - */ -export const roleMemberResponse = z.object({ - roleId: z.string(), - memberId: z.string(), - memberName: z.string(), - withAdminOption: z.boolean(), - createdAt: z.string(), -}); - -export type RoleMemberResponse = z.infer; - -/** - * Role info response — used in listForUser result. - */ -export const roleInfoResponse = z.object({ - id: z.string(), - name: z.string(), - withAdminOption: z.boolean(), -}); - -export type RoleInfoResponse = z.infer; - -/** - * role.addMember result. - */ -export const roleAddMemberResult = z.object({ - added: z.boolean(), -}); - -export type RoleAddMemberResult = z.infer; - -/** - * role.removeMember result. - */ -export const roleRemoveMemberResult = z.object({ - removed: z.boolean(), -}); - -export type RoleRemoveMemberResult = z.infer; - -/** - * role.listMembers result. - */ -export const roleListMembersResult = z.object({ - members: z.array(roleMemberResponse), -}); - -export type RoleListMembersResult = z.infer; - -/** - * role.listForUser result. - */ -export const roleListForUserResult = z.object({ - roles: z.array(roleInfoResponse), -}); - -export type RoleListForUserResult = z.infer; diff --git a/packages/protocol/engine/user.ts b/packages/protocol/engine/user.ts deleted file mode 100644 index ed2ee88..0000000 --- a/packages/protocol/engine/user.ts +++ /dev/null @@ -1,116 +0,0 @@ -/** - * User method schemas — params and results for user.* RPC methods. - */ -import { z } from "zod"; -import { uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * user.create params. - */ -export const userCreateParams = z.object({ - id: uuidv7Schema.optional().nullable(), - name: z.string().min(1, "name is required"), - identityId: uuidv7Schema.optional().nullable(), - canLogin: z.boolean().optional(), - superuser: z.boolean().optional(), - createrole: z.boolean().optional(), -}); - -export type UserCreateParams = z.infer; - -/** - * user.get params. - */ -export const userGetParams = z.object({ - id: uuidv7Schema, -}); - -export type UserGetParams = z.infer; - -/** - * user.getByName params. - */ -export const userGetByNameParams = z.object({ - name: z.string().min(1), -}); - -export type UserGetByNameParams = z.infer; - -/** - * user.list params. - */ -export const userListParams = z.object({ - canLogin: z.boolean().optional(), -}); - -export type UserListParams = z.infer; - -/** - * user.rename params. - */ -export const userRenameParams = z.object({ - id: uuidv7Schema, - name: z.string().min(1, "name is required"), -}); - -export type UserRenameParams = z.infer; - -/** - * user.delete params. - */ -export const userDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type UserDeleteParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single user response — returned by create, get, getByName. - */ -export const userResponse = z.object({ - id: z.string(), - name: z.string(), - identityId: z.string().nullable(), - canLogin: z.boolean(), - superuser: z.boolean(), - createrole: z.boolean(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type UserResponse = z.infer; - -/** - * user.list result. - */ -export const userListResult = z.object({ - users: z.array(userResponse), -}); - -export type UserListResult = z.infer; - -/** - * user.rename result. - */ -export const userRenameResult = z.object({ - renamed: z.boolean(), -}); - -export type UserRenameResult = z.infer; - -/** - * user.delete result. - */ -export const userDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type UserDeleteResult = z.infer; diff --git a/packages/protocol/index.ts b/packages/protocol/index.ts index 2eadc5c..67377a1 100644 --- a/packages/protocol/index.ts +++ b/packages/protocol/index.ts @@ -5,17 +5,15 @@ * client and server. Both the server (validation) and client libraries * (type inference + optional response validation) import from here. * - * Two RPC endpoints, two contracts: - * - Engine RPC (POST /api/v1/engine/rpc) — API key auth, 30 methods - * - Accounts RPC (POST /api/v1/accounts/rpc) — session token auth, 19 methods + * RPC endpoints / contracts: + * - Memory RPC (POST /api/v1/memory/rpc) — session or api-key auth; the memory + * data plane (./memory) + the space management contract (./space). + * - User RPC (POST /api/v1/user/rpc) — session auth; whoami + agent + space + * discovery (./user). */ -// Accounts RPC contract + all accounts schemas -export * from "./accounts/index.ts"; // Device flow auth schemas export * from "./auth/device-flow.ts"; -// Engine RPC contract + all engine schemas -export * from "./engine/index.ts"; // Error codes and AppError export * from "./errors.ts"; // Shared field validators @@ -24,5 +22,7 @@ export * from "./fields.ts"; export * from "./headers.ts"; // JSON-RPC 2.0 envelope types export * from "./jsonrpc.ts"; +// Memory data-plane schemas +export * from "./memory.ts"; // Version compatibility schemas export * from "./version.ts"; diff --git a/packages/protocol/engine/memory.ts b/packages/protocol/memory.ts similarity index 99% rename from packages/protocol/engine/memory.ts rename to packages/protocol/memory.ts index da65f12..4347870 100644 --- a/packages/protocol/engine/memory.ts +++ b/packages/protocol/memory.ts @@ -10,7 +10,7 @@ import { treeFilterSchema, treePathSchema, uuidv7Schema, -} from "../fields.ts"; +} from "./fields.ts"; // ============================================================================= // Params Schemas diff --git a/packages/protocol/package.json b/packages/protocol/package.json index c3044e7..32638b8 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -39,15 +39,10 @@ "types": "./version.ts", "import": "./dist/version.js" }, - "./engine": { - "bun": "./engine/index.ts", - "types": "./engine/index.ts", - "import": "./dist/engine/index.js" - }, - "./engine/*": { - "bun": "./engine/*.ts", - "types": "./engine/*.ts", - "import": "./dist/engine/*.js" + "./memory": { + "bun": "./memory.ts", + "types": "./memory.ts", + "import": "./dist/memory.js" }, "./space": { "bun": "./space/index.ts", @@ -69,16 +64,6 @@ "types": "./user/*.ts", "import": "./dist/user/*.js" }, - "./accounts": { - "bun": "./accounts/index.ts", - "types": "./accounts/index.ts", - "import": "./dist/accounts/index.js" - }, - "./accounts/*": { - "bun": "./accounts/*.ts", - "types": "./accounts/*.ts", - "import": "./dist/accounts/*.js" - }, "./auth/*": { "bun": "./auth/*.ts", "types": "./auth/*.ts", @@ -88,15 +73,13 @@ "files": [ "dist", "*.ts", - "engine/*.ts", "space/*.ts", "user/*.ts", - "accounts/*.ts", "auth/*.ts", "!*.test.ts" ], "scripts": { - "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts headers.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts space/index.ts space/principal.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts user/whoami.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", + "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts headers.ts version.ts memory.ts space/index.ts space/principal.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts user/whoami.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", "build:types": "tsc -p tsconfig.build.json", "build": "../../bun run build:js && ../../bun run build:types", "prepublishOnly": "../../bun run build" diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 001db26..2218404 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -32,7 +32,7 @@ import type { MemoryTreeParams, MemoryTreeResult, MemoryUpdateParams, -} from "@memory.build/protocol/engine/memory"; +} from "@memory.build/protocol/memory"; import { memoryBatchCreateParams, memoryCountTreeParams, @@ -44,7 +44,7 @@ import { memorySearchParams, memoryTreeParams, memoryUpdateParams, -} from "@memory.build/protocol/engine/memory"; +} from "@memory.build/protocol/memory"; import { AppError } from "../errors"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; From 2d50233942194705bc85490408a445e9a83dc28f Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 21:40:06 +0200 Subject: [PATCH 065/156] chore: rename ENGINE_*/WORKER_ENGINE_* env vars; drop dead master-key script (5F) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the legacy engine/accounts paths are gone, the single application pool no longer warrants "engine" naming. - ENGINE_DATABASE_URL -> DATABASE_URL; ENGINE_POOL_* -> DB_POOL_*; WORKER_ENGINE_DATABASE_URL -> WORKER_DATABASE_URL; WORKER_ENGINE_POOL_* -> WORKER_DB_POOL_*; WORKER_ENGINE_*_TIMEOUT -> WORKER_DB_*_TIMEOUT. Updated server/index.ts (env + internal consts + doc block), scripts/setup.ts (one DB now), scripts/migrate-db.ts (drop the ENGINE_DATABASE_URL fallback), DEVELOPMENT.md, and .env.sample (dropped the ACCOUNTS_* + legacy ENGINE_*_TIMEOUT blocks). - delete scripts/generate-master-key.ts (ACCOUNTS_MASTER_KEY — crypto was removed earlier; the script was unreferenced). DEPLOYMENT NOTE: the running server now requires DATABASE_URL (was ENGINE_DATABASE_URL) and no longer reads ACCOUNTS_DATABASE_URL — deploy env/ secrets must be updated in lockstep. typecheck + lint + unit (642) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.sample | 115 +++++++--------------------- DEVELOPMENT.md | 8 +- packages/server/index.ts | 132 ++++++++++++++++----------------- scripts/generate-master-key.ts | 11 --- scripts/migrate-db.ts | 8 +- scripts/setup.ts | 10 +-- 6 files changed, 97 insertions(+), 187 deletions(-) delete mode 100644 scripts/generate-master-key.ts diff --git a/.env.sample b/.env.sample index 9237129..61488f6 100644 --- a/.env.sample +++ b/.env.sample @@ -7,13 +7,9 @@ # Required # ----------------------------------------------------------------------------- -# PostgreSQL connection string for the accounts database -# (stores identities, orgs, engines, sessions, API keys) -ACCOUNTS_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me - -# PostgreSQL connection string for the engine database -# (stores memories — each engine gets its own schema) -ENGINE_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me +# PostgreSQL connection string for the application database. One database holds +# the auth + core control plane and every per-space me_ schema. +DATABASE_URL=postgres://postgres:postgres@localhost:5432/me # Public base URL for OAuth callbacks API_BASE_URL=http://localhost:3000 @@ -47,8 +43,9 @@ GOOGLE_CLIENT_SECRET= # Raise for memory.batchCreate-heavy workloads (e.g. transcript imports). # MAX_REQUEST_BODY_BYTES= -# Schema name in the accounts database -# ACCOUNTS_SCHEMA=accounts +# Schema names +# AUTH_SCHEMA=auth +# CORE_SCHEMA=core # Cron schedule for cleaning up expired device authorizations (UTC) # DEVICE_FLOW_CLEANUP_CRON=*/15 * * * * @@ -69,60 +66,20 @@ GOOGLE_CLIENT_SECRET= # LOGFIRE_SCRUBBING=false # ----------------------------------------------------------------------------- -# Optional — Accounts Database Connection Pool +# Optional — Database Connection Pool # ----------------------------------------------------------------------------- # Maximum connections in pool -# ACCOUNTS_POOL_MAX=10 +# DB_POOL_MAX=20 # Close idle pooled connections after N seconds -# ACCOUNTS_POOL_IDLE_REAP_SECONDS=300 +# DB_POOL_IDLE_REAP_SECONDS=300 # Max connection lifetime in seconds (0 = forever) -# ACCOUNTS_POOL_MAX_LIFETIME=0 +# DB_POOL_MAX_LIFETIME=0 # Timeout for establishing new connections (seconds) -# ACCOUNTS_POOL_CONNECTION_TIMEOUT=30 - -# PostgreSQL statement timeout for accounts DB transactions -# ACCOUNTS_STATEMENT_TIMEOUT=25s - -# PostgreSQL lock wait timeout for accounts DB transactions -# ACCOUNTS_LOCK_TIMEOUT=5s - -# PostgreSQL transaction timeout for accounts DB transactions -# ACCOUNTS_TRANSACTION_TIMEOUT=30s - -# PostgreSQL idle-in-transaction timeout for accounts DB transactions -# ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s - -# ----------------------------------------------------------------------------- -# Optional — Engine Database Connection Pool -# ----------------------------------------------------------------------------- - -# Maximum connections in pool -# ENGINE_POOL_MAX=20 - -# Close idle pooled connections after N seconds -# ENGINE_POOL_IDLE_REAP_SECONDS=300 - -# Max connection lifetime in seconds (0 = forever) -# ENGINE_POOL_MAX_LIFETIME=0 - -# Timeout for establishing new connections (seconds) -# ENGINE_POOL_CONNECTION_TIMEOUT=30 - -# PostgreSQL statement timeout for engine DB transactions -# ENGINE_STATEMENT_TIMEOUT=25s - -# PostgreSQL lock wait timeout for engine DB transactions -# ENGINE_LOCK_TIMEOUT=5s - -# PostgreSQL transaction timeout for engine DB transactions -# ENGINE_TRANSACTION_TIMEOUT=30s - -# PostgreSQL idle-in-transaction timeout for engine DB transactions -# ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s +# DB_POOL_CONNECTION_TIMEOUT=30 # ----------------------------------------------------------------------------- # Optional — Embedding @@ -159,45 +116,23 @@ GOOGLE_CLIENT_SECRET= # Max error backoff (ms) # WORKER_MAX_BACKOFF_MS=60000 -# Engine re-discovery interval (ms) +# Space re-discovery interval (ms) # WORKER_REFRESH_INTERVAL_MS=60000 # ----------------------------------------------------------------------------- -# Optional — Embedding Worker Engine Database +# Optional — Embedding Worker Database Pool # ----------------------------------------------------------------------------- +# The embedding worker uses a dedicated pool. Each setting defaults to the +# corresponding application-pool value (or DATABASE_URL) when unset. -# PostgreSQL connection string for embedding worker engine traffic. -# Defaults to ENGINE_DATABASE_URL when unset. -# WORKER_ENGINE_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me - -# Maximum connections in the dedicated embedding worker engine pool. -# Defaults to max(WORKER_COUNT, 1) when unset. -# WORKER_ENGINE_POOL_MAX=2 - -# Close idle embedding worker engine connections after N seconds. -# Defaults to ENGINE_POOL_IDLE_REAP_SECONDS when unset. -# WORKER_ENGINE_POOL_IDLE_REAP_SECONDS=300 - -# Max embedding worker engine connection lifetime in seconds (0 = forever). -# Defaults to ENGINE_POOL_MAX_LIFETIME when unset. -# WORKER_ENGINE_POOL_MAX_LIFETIME=0 - -# Timeout for establishing embedding worker engine connections (seconds). -# Defaults to ENGINE_POOL_CONNECTION_TIMEOUT when unset. -# WORKER_ENGINE_POOL_CONNECTION_TIMEOUT=30 - -# PostgreSQL statement timeout for embedding worker engine DB transactions -# Defaults to ENGINE_STATEMENT_TIMEOUT when unset. -# WORKER_ENGINE_STATEMENT_TIMEOUT=25s - -# PostgreSQL lock wait timeout for embedding worker engine DB transactions -# Defaults to ENGINE_LOCK_TIMEOUT when unset. -# WORKER_ENGINE_LOCK_TIMEOUT=5s - -# PostgreSQL transaction timeout for embedding worker engine DB transactions -# Defaults to ENGINE_TRANSACTION_TIMEOUT when unset. -# WORKER_ENGINE_TRANSACTION_TIMEOUT=30s +# WORKER_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me +# WORKER_DB_POOL_MAX=2 +# WORKER_DB_POOL_IDLE_REAP_SECONDS=300 +# WORKER_DB_POOL_MAX_LIFETIME=0 +# WORKER_DB_POOL_CONNECTION_TIMEOUT=30 -# PostgreSQL idle-in-transaction timeout for embedding worker engine DB transactions -# Defaults to ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT when unset. -# WORKER_ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s +# Per-transaction timeouts for the worker pool (default to built-in values). +# WORKER_DB_STATEMENT_TIMEOUT=25s +# WORKER_DB_LOCK_TIMEOUT=5s +# WORKER_DB_TRANSACTION_TIMEOUT=30s +# WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5a0a9f4..3425fcf 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -52,11 +52,10 @@ cp .env.sample .env #### Required variables -**Database connections** — both point to the local Docker Postgres, but use separate databases: +**Database connection** — one database holds the `auth` + `core` control plane and every per-space `me_` schema: ``` -ACCOUNTS_DATABASE_URL=postgres://postgres@localhost:5432/accounts -ENGINE_DATABASE_URL=postgres://postgres@localhost:5432/shard1 +DATABASE_URL=postgres://postgres@localhost:5432/memory_engine ``` @@ -112,8 +111,7 @@ GITHUB_CLIENT_SECRET=... ```bash # Database -ACCOUNTS_DATABASE_URL=postgres://postgres@localhost:5432/accounts -ENGINE_DATABASE_URL=postgres://postgres@localhost:5432/shard1 +DATABASE_URL=postgres://postgres@localhost:5432/memory_engine # Server API_BASE_URL=http://localhost:3000 diff --git a/packages/server/index.ts b/packages/server/index.ts index 361787a..af16023 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -62,7 +62,7 @@ configure({ // ============================================================================= // // Required: -// ENGINE_DATABASE_URL - PostgreSQL connection string for the database that +// DATABASE_URL - PostgreSQL connection string for the database that // holds the auth + core control plane and every // per-space me_ schema (one DB, one pool) // API_BASE_URL - Public URL for OAuth callbacks @@ -74,25 +74,21 @@ configure({ // CORE_SCHEMA - Core control-plane schema name (default: "core") // // Connection Pool - Database: -// ENGINE_POOL_MAX - Max connections (default: 20) -// ENGINE_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: 300) -// ENGINE_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: 0) -// ENGINE_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: 30) -// ENGINE_STATEMENT_TIMEOUT - Per-engine-query timeout (default: 25s) -// ENGINE_LOCK_TIMEOUT - Per-engine-lock wait timeout (default: 5s) -// ENGINE_TRANSACTION_TIMEOUT - Per-engine-transaction timeout (default: 30s) -// ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Idle-in-transaction timeout (default: 30s) +// DB_POOL_MAX - Max connections (default: 20) +// DB_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: 300) +// DB_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: 0) +// DB_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: 30) // -// Embedding Worker Engine Database: -// WORKER_ENGINE_DATABASE_URL - PostgreSQL connection string for worker engine traffic (default: ENGINE_DATABASE_URL) -// WORKER_ENGINE_POOL_MAX - Max worker engine connections (default: WORKER_COUNT) -// WORKER_ENGINE_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: ENGINE_POOL_IDLE_REAP_SECONDS) -// WORKER_ENGINE_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: ENGINE_POOL_MAX_LIFETIME) -// WORKER_ENGINE_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: ENGINE_POOL_CONNECTION_TIMEOUT) -// WORKER_ENGINE_STATEMENT_TIMEOUT - Worker engine query timeout (default: ENGINE_STATEMENT_TIMEOUT) -// WORKER_ENGINE_LOCK_TIMEOUT - Worker engine lock wait timeout (default: ENGINE_LOCK_TIMEOUT) -// WORKER_ENGINE_TRANSACTION_TIMEOUT - Worker engine transaction timeout (default: ENGINE_TRANSACTION_TIMEOUT) -// WORKER_ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Worker engine idle-in-transaction timeout (default: ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT) +// Embedding Worker Database: +// WORKER_DATABASE_URL - PostgreSQL connection string for worker traffic (default: DATABASE_URL) +// WORKER_DB_POOL_MAX - Max worker connections (default: WORKER_COUNT) +// WORKER_DB_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: DB_POOL_IDLE_REAP_SECONDS) +// WORKER_DB_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: DB_POOL_MAX_LIFETIME) +// WORKER_DB_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: DB_POOL_CONNECTION_TIMEOUT) +// WORKER_DB_STATEMENT_TIMEOUT - Worker query timeout (default: built-in worker default) +// WORKER_DB_LOCK_TIMEOUT - Worker lock wait timeout (default: built-in worker default) +// WORKER_DB_TRANSACTION_TIMEOUT - Worker transaction timeout (default: built-in worker default) +// WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Worker idle-in-transaction timeout (default: built-in worker default) // // Cleanup: // DEVICE_FLOW_CLEANUP_CRON - Cron schedule for cleaning up expired device auths @@ -104,7 +100,7 @@ configure({ // WORKER_LOCK_DURATION - PostgreSQL interval for claim lock (default: "5 minutes") // WORKER_IDLE_DELAY_MS - Poll interval when idle in ms (default: 10000) // WORKER_MAX_BACKOFF_MS - Max error backoff in ms (default: 60000) -// WORKER_REFRESH_INTERVAL_MS - Engine re-discovery interval in ms (default: 60000) +// WORKER_REFRESH_INTERVAL_MS - Space re-discovery interval in ms (default: 60000) // // ============================================================================= @@ -128,9 +124,9 @@ function parseIntEnv( const port = process.env.PORT || 3000; -const engineDatabaseUrl = process.env.ENGINE_DATABASE_URL; -if (!engineDatabaseUrl) { - throw new Error("ENGINE_DATABASE_URL environment variable is required"); +const databaseUrl = process.env.DATABASE_URL; +if (!databaseUrl) { + throw new Error("DATABASE_URL environment variable is required"); } const apiBaseUrl = process.env.API_BASE_URL; @@ -152,62 +148,60 @@ const workerCount = parseIntEnv( ); // Connection pool settings - database -const enginePoolMax = parseIntEnv( - "ENGINE_POOL_MAX", - process.env.ENGINE_POOL_MAX || "", +const dbPoolMax = parseIntEnv( + "DB_POOL_MAX", + process.env.DB_POOL_MAX || "", "20", ); -const enginePoolIdleReapSeconds = parseIntEnv( - "ENGINE_POOL_IDLE_REAP_SECONDS", - process.env.ENGINE_POOL_IDLE_REAP_SECONDS || "", +const dbPoolIdleReapSeconds = parseIntEnv( + "DB_POOL_IDLE_REAP_SECONDS", + process.env.DB_POOL_IDLE_REAP_SECONDS || "", "300", ); -const enginePoolMaxLifetime = parseIntEnv( - "ENGINE_POOL_MAX_LIFETIME", - process.env.ENGINE_POOL_MAX_LIFETIME || "", +const dbPoolMaxLifetime = parseIntEnv( + "DB_POOL_MAX_LIFETIME", + process.env.DB_POOL_MAX_LIFETIME || "", "0", ); -const enginePoolConnectionTimeout = parseIntEnv( - "ENGINE_POOL_CONNECTION_TIMEOUT", - process.env.ENGINE_POOL_CONNECTION_TIMEOUT || "", +const dbPoolConnectionTimeout = parseIntEnv( + "DB_POOL_CONNECTION_TIMEOUT", + process.env.DB_POOL_CONNECTION_TIMEOUT || "", "30", ); -// Connection pool settings - Embedding worker engine database -const workerEngineDatabaseUrl = - process.env.WORKER_ENGINE_DATABASE_URL || engineDatabaseUrl; -const workerEnginePoolMax = parseIntEnv( - "WORKER_ENGINE_POOL_MAX", - process.env.WORKER_ENGINE_POOL_MAX || "", +// Connection pool settings - embedding worker database +const workerDatabaseUrl = process.env.WORKER_DATABASE_URL || databaseUrl; +const workerDbPoolMax = parseIntEnv( + "WORKER_DB_POOL_MAX", + process.env.WORKER_DB_POOL_MAX || "", String(Math.max(workerCount, 1)), ); -const workerEnginePoolIdleReapSeconds = parseIntEnv( - "WORKER_ENGINE_POOL_IDLE_REAP_SECONDS", - process.env.WORKER_ENGINE_POOL_IDLE_REAP_SECONDS || "", - String(enginePoolIdleReapSeconds), +const workerDbPoolIdleReapSeconds = parseIntEnv( + "WORKER_DB_POOL_IDLE_REAP_SECONDS", + process.env.WORKER_DB_POOL_IDLE_REAP_SECONDS || "", + String(dbPoolIdleReapSeconds), ); -const workerEnginePoolMaxLifetime = parseIntEnv( - "WORKER_ENGINE_POOL_MAX_LIFETIME", - process.env.WORKER_ENGINE_POOL_MAX_LIFETIME || "", - String(enginePoolMaxLifetime), +const workerDbPoolMaxLifetime = parseIntEnv( + "WORKER_DB_POOL_MAX_LIFETIME", + process.env.WORKER_DB_POOL_MAX_LIFETIME || "", + String(dbPoolMaxLifetime), ); -const workerEnginePoolConnectionTimeout = parseIntEnv( - "WORKER_ENGINE_POOL_CONNECTION_TIMEOUT", - process.env.WORKER_ENGINE_POOL_CONNECTION_TIMEOUT || "", - String(enginePoolConnectionTimeout), +const workerDbPoolConnectionTimeout = parseIntEnv( + "WORKER_DB_POOL_CONNECTION_TIMEOUT", + process.env.WORKER_DB_POOL_CONNECTION_TIMEOUT || "", + String(dbPoolConnectionTimeout), ); const workerTimeouts: WorkerTimeouts = { statementTimeout: - process.env.WORKER_ENGINE_STATEMENT_TIMEOUT ?? + process.env.WORKER_DB_STATEMENT_TIMEOUT ?? DEFAULT_WORKER_TIMEOUTS.statementTimeout, lockTimeout: - process.env.WORKER_ENGINE_LOCK_TIMEOUT ?? - DEFAULT_WORKER_TIMEOUTS.lockTimeout, + process.env.WORKER_DB_LOCK_TIMEOUT ?? DEFAULT_WORKER_TIMEOUTS.lockTimeout, transactionTimeout: - process.env.WORKER_ENGINE_TRANSACTION_TIMEOUT ?? + process.env.WORKER_DB_TRANSACTION_TIMEOUT ?? DEFAULT_WORKER_TIMEOUTS.transactionTimeout, idleInTransactionSessionTimeout: - process.env.WORKER_ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? + process.env.WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? DEFAULT_WORKER_TIMEOUTS.idleInTransactionSessionTimeout, }; @@ -215,7 +209,7 @@ const workerTimeouts: WorkerTimeouts = { // Embedding Config // ============================================================================= // -// Model and dimensions are hardcoded - all engines use the same embedding model. +// Model and dimensions are hardcoded - all spaces use the same embedding model. // Only the API key is configurable via environment variable. // // Required: @@ -298,21 +292,21 @@ if (configuredProviders.length === 0) { // Dedicated worker pool (postgres.js) — the embedding worker processes the // per-space me_ schemas. -const workerDb = postgres(workerEngineDatabaseUrl, { - max: workerEnginePoolMax, - idle_timeout: workerEnginePoolIdleReapSeconds, - max_lifetime: workerEnginePoolMaxLifetime, - connect_timeout: workerEnginePoolConnectionTimeout, +const workerDb = postgres(workerDatabaseUrl, { + max: workerDbPoolMax, + idle_timeout: workerDbPoolIdleReapSeconds, + max_lifetime: workerDbPoolMaxLifetime, + connect_timeout: workerDbPoolConnectionTimeout, onnotice: () => {}, }); // The single application pool (postgres.js): the auth + core control plane and // the per-space me_ data schemas all live in one database, one pool. -const db = postgres(engineDatabaseUrl, { - max: enginePoolMax, - idle_timeout: enginePoolIdleReapSeconds, - max_lifetime: enginePoolMaxLifetime, - connect_timeout: enginePoolConnectionTimeout, +const db = postgres(databaseUrl, { + max: dbPoolMax, + idle_timeout: dbPoolIdleReapSeconds, + max_lifetime: dbPoolMaxLifetime, + connect_timeout: dbPoolConnectionTimeout, onnotice: () => {}, }); diff --git a/scripts/generate-master-key.ts b/scripts/generate-master-key.ts deleted file mode 100644 index f7433b0..0000000 --- a/scripts/generate-master-key.ts +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bun -/** - * Generate a random 32-byte (256-bit) master key for ACCOUNTS_MASTER_KEY. - * - * Usage: - * ./bun scripts/generate-master-key.ts - */ -import { randomBytes } from "node:crypto"; - -const key = randomBytes(32).toString("hex"); -console.log(key); diff --git a/scripts/migrate-db.ts b/scripts/migrate-db.ts index 312e283..20c1b53 100644 --- a/scripts/migrate-db.ts +++ b/scripts/migrate-db.ts @@ -18,7 +18,7 @@ function usage(): string { return `Usage: ./bun run migrate:db [all|core|space-db|space] Environment: - DATABASE_URL Postgres connection string. Falls back to ENGINE_DATABASE_URL, then ${DEFAULT_DATABASE_URL} + DATABASE_URL Postgres connection string. Defaults to ${DEFAULT_DATABASE_URL} SPACE_SLUG Space slug to migrate. Defaults to ${DEFAULT_SPACE_SLUG} Modes: @@ -49,11 +49,7 @@ function parseMode(arg: string | undefined): Mode { } function databaseUrl(): string { - return ( - process.env.DATABASE_URL ?? - process.env.ENGINE_DATABASE_URL ?? - DEFAULT_DATABASE_URL - ); + return process.env.DATABASE_URL ?? DEFAULT_DATABASE_URL; } async function main(): Promise { diff --git a/scripts/setup.ts b/scripts/setup.ts index aff03ca..f02a336 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -11,7 +11,7 @@ * * Prerequisites: * 1. Postgres running (`./bun run pg`) - * 2. .env filled in (ACCOUNTS_DATABASE_URL, ENGINE_DATABASE_URL) + * 2. .env filled in (DATABASE_URL) * * Usage: * ./bun run setup @@ -32,8 +32,7 @@ function requireEnv(name: string): string { return value; } -const ACCOUNTS_DATABASE_URL = requireEnv("ACCOUNTS_DATABASE_URL"); -const ENGINE_DATABASE_URL = requireEnv("ENGINE_DATABASE_URL"); +const DATABASE_URL = requireEnv("DATABASE_URL"); // ============================================================================= // Database creation @@ -75,9 +74,8 @@ async function main() { console.log("=== Memory Engine: Local Dev Setup ==="); console.log(""); - console.log("Ensuring databases exist..."); - await ensureDatabase(ACCOUNTS_DATABASE_URL); - await ensureDatabase(ENGINE_DATABASE_URL); + console.log("Ensuring database exists..."); + await ensureDatabase(DATABASE_URL); console.log(""); console.log("Done! Start the server to run bootstrap + migrations:"); From 374c0da489c0e856b60317ae424ef890062738d8 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 4 Jun 2026 21:41:42 +0200 Subject: [PATCH 066/156] chore: tidy residual engine naming (5F follow-up) - transport.test.ts: dummy RPC path /api/v1/engine/rpc -> /api/v1/memory/rpc (the transport is path-agnostic; the engine endpoint no longer exists). - packages/web: rename the client instance memoryEngineClient -> memoryClient. No behavior change. typecheck + lint + unit (642) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/client/transport.test.ts | 2 +- packages/web/src/api/client.ts | 2 +- packages/web/src/api/queries.ts | 16 ++++++++-------- .../src/components/dialogs/DeleteTreeDialog.tsx | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/client/transport.test.ts b/packages/client/transport.test.ts index 4a4e791..d9d12b4 100644 --- a/packages/client/transport.test.ts +++ b/packages/client/transport.test.ts @@ -45,7 +45,7 @@ function captureFetch(): { const baseConfig = { url: "https://api.example.com", - path: "/api/v1/engine/rpc", + path: "/api/v1/memory/rpc", timeout: 5_000, retries: 0, } satisfies Omit; diff --git a/packages/web/src/api/client.ts b/packages/web/src/api/client.ts index b7895a4..6255867 100644 --- a/packages/web/src/api/client.ts +++ b/packages/web/src/api/client.ts @@ -8,7 +8,7 @@ */ import { createMemoryClient } from "@memory.build/client"; -export const memoryEngineClient = createMemoryClient({ +export const memoryClient = createMemoryClient({ url: "", rpcPath: "/rpc", retries: 0, diff --git a/packages/web/src/api/queries.ts b/packages/web/src/api/queries.ts index 2e8739f..b3de309 100644 --- a/packages/web/src/api/queries.ts +++ b/packages/web/src/api/queries.ts @@ -15,7 +15,7 @@ import { useQuery, useQueryClient, } from "@tanstack/react-query"; -import { memoryEngineClient } from "./client.ts"; +import { memoryClient } from "./client.ts"; const SEARCH_LIMIT = 1000; @@ -49,7 +49,7 @@ export function useMemories(params: MemorySearchParams, enabled = true) { return useQuery({ enabled, queryKey: ["memories", normalized], - queryFn: () => memoryEngineClient.memory.search(normalized), + queryFn: () => memoryClient.memory.search(normalized), }); } @@ -62,7 +62,7 @@ export function useMemories(params: MemorySearchParams, enabled = true) { export function useTree() { return useQuery({ queryKey: ["memory-tree"], - queryFn: () => memoryEngineClient.memory.tree(), + queryFn: () => memoryClient.memory.tree(), }); } @@ -78,7 +78,7 @@ export function useMemoriesAtExactPath(path: string, enabled: boolean) { enabled, queryKey: ["memories-at-exact-path", path], queryFn: () => - memoryEngineClient.memory.search({ + memoryClient.memory.search({ tree: exactTreeLquery(path), limit: SEARCH_LIMIT, }), @@ -94,7 +94,7 @@ export function useMemory(id: string | null) { return useQuery({ enabled: id !== null, queryKey: ["memory", id], - queryFn: () => memoryEngineClient.memory.get({ id: id as string }), + queryFn: () => memoryClient.memory.get({ id: id as string }), }); } @@ -104,7 +104,7 @@ export function useMemory(id: string | null) { export function useUpdateMemory(queryClient: QueryClient) { return useMutation({ mutationFn: (params: MemoryUpdateParams) => - memoryEngineClient.memory.update(params), + memoryClient.memory.update(params), onSuccess: (memory) => { invalidateTreeQueries(queryClient); queryClient.setQueryData(["memory", memory.id], memory); @@ -117,7 +117,7 @@ export function useUpdateMemory(queryClient: QueryClient) { */ export function useDeleteMemory(queryClient: QueryClient) { return useMutation({ - mutationFn: (id: string) => memoryEngineClient.memory.delete({ id }), + mutationFn: (id: string) => memoryClient.memory.delete({ id }), onSuccess: (_result, id) => { invalidateTreeQueries(queryClient); queryClient.removeQueries({ queryKey: ["memory", id] }); @@ -132,7 +132,7 @@ export function useDeleteMemory(queryClient: QueryClient) { export function useDeleteTree(queryClient: QueryClient) { return useMutation({ mutationFn: (args: { tree: string; dryRun?: boolean }) => - memoryEngineClient.memory.deleteTree(args), + memoryClient.memory.deleteTree(args), onSuccess: (_result, args) => { if (!args.dryRun) { invalidateTreeQueries(queryClient); diff --git a/packages/web/src/components/dialogs/DeleteTreeDialog.tsx b/packages/web/src/components/dialogs/DeleteTreeDialog.tsx index d364774..42c6ee7 100644 --- a/packages/web/src/components/dialogs/DeleteTreeDialog.tsx +++ b/packages/web/src/components/dialogs/DeleteTreeDialog.tsx @@ -6,7 +6,7 @@ * be removed. On confirm, re-issues the call with `dryRun: false`. */ import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { memoryEngineClient } from "../../api/client.ts"; +import { memoryClient } from "../../api/client.ts"; import { useDeleteTree } from "../../api/queries.ts"; import { useSelection } from "../../store/selection.ts"; import { useUi } from "../../store/ui.ts"; @@ -26,7 +26,7 @@ export function DeleteTreeDialog() { enabled: treePath !== null, queryKey: ["deleteTreeDryRun", treePath], queryFn: () => - memoryEngineClient.memory.deleteTree({ + memoryClient.memory.deleteTree({ tree: treePath as string, dryRun: true, }), From 512343c5c5f173f5ddd22bcb58fd6baec7704ea1 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 09:13:15 +0200 Subject: [PATCH 067/156] docs: update CLAUDE.md to the post-migration design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the project doc for the new model now that Phases 1–5 are done: - Quick Reference: three schemas (auth/core/me_) in one DB / one postgres.js pool; tree_access (no RLS); the two RPC endpoints (memory + user) with session-or-api-key + X-Me-Space; the new CLI command surface. - New "Principals, members, spaces" terminology section (principal = u|a|g; member/memberId = u|a; immutable slug vs renamable name; transitive admin). - Project Structure: real package list (auth, database, web, claude-plugin, engine=core+space, …); dropped the obsolete `hosted` note (accounts deleted). - Key Design Decisions: bakes in the choices made in review — api keys are never stored in the credentials file (ME_API_KEY only), header constants in protocol/headers, space.rename (slug immutable), access via tree_access. - Driver section: postgres.js is the runtime driver (migration done; only the dev setup.ts helper still uses Bun.SQL); kept the gotchas. Also reap orphaned `auth_test_*` schemas in clean-test-schemas.ts (auth tests now provision them alongside core_test_*/metest_*) and document the third prefix. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 150 ++++++++++++++++++---------------- scripts/clean-test-schemas.ts | 21 +++-- 2 files changed, 94 insertions(+), 77 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6870ea5..a8ede73 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ All project documentation lives in `docs/`: - [Getting Started](docs/getting-started.md) -- install, login, first memory - [Core Concepts](docs/concepts.md) -- memories, tree paths, metadata, search modes - [File Formats](docs/formats.md) -- JSON, YAML, Markdown, NDJSON import/export schemas -- [Access Control](docs/access-control.md) -- users, roles, grants, ownership +- [Access Control](docs/access-control.md) -- principals, groups, tree-access grants - [Memory Packs](docs/memory-packs.md) -- pre-built knowledge collections - [MCP Integration](docs/mcp-integration.md) -- connecting AI agents - [CLI Reference](docs/cli/) -- full command reference @@ -17,34 +17,56 @@ All project documentation lives in `docs/`: Read the relevant docs before starting work on a subsystem. +> **Note**: the authoritative summary of the current model (principals / spaces / +> the auth+core+space schemas) is in this file. Some `docs/` pages still describe +> the retired engine/org/role model and may lag — trust this file when they +> disagree, and fix the docs as you touch them. + ## Quick Reference -- **Tech stack**: Bun, TypeScript, PostgreSQL 18 (pgvector, pg_textsearch, ltree, JSONB) -- **Core schema**: Single table `memory` per engine schema (`me_`) -- content, meta (JSONB), tree (ltree), temporal (tstzrange), embedding (halfvec(1536)) -- **Search**: Hybrid BM25 + semantic via Reciprocal Rank Fusion -- **API**: JSON-RPC 2.0 over HTTP -- engine RPC (`/api/v1/engine/rpc`) and accounts RPC (`/api/v1/accounts/rpc`), plus REST auth endpoints (OAuth device flow) -- **Auth**: Tree-grant RBAC with PostgreSQL RLS; OAuth (GitHub, Google) for hosted accounts -- **Embedding**: Vercel AI SDK; OpenAI `text-embedding-3-small` (1536-dim) in production; Ollama supported for local dev -- **CLI**: `me` binary (login, logout, whoami, org, engine, invitation, memory, mcp, user, grant, role, owner, apikey, pack) +- **Tech stack**: Bun, TypeScript, PostgreSQL 18 (pgvector/halfvec, pg_textsearch BM25, ltree, citext, JSONB), **postgres.js** driver. One database, one pool. +- **Schemas** (three, one database): `auth` (better-auth-shaped: `users`, `sessions`, `accounts`, `device_authorization`), `core` (control plane: `principal`, `space`, `principal_space`, `group_member`, `tree_access`, `api_key`), and per-space `me_` (data plane: the single `memory` table). `auth.users.id == core.principal.id` for user principals. +- **Memory table** (per space): `content`, `meta` (JSONB), `tree` (ltree), `temporal` (tstzrange), `embedding` (halfvec(1536)). +- **Search**: hybrid BM25 + semantic via Reciprocal Rank Fusion, computed in SQL functions. +- **Access**: no RLS. `core.build_tree_access(principalId, spaceId)` produces a `_tree_access` jsonb (rows of `tree_path` + `access`) passed into the space SQL functions (`search_memory`, `get_memory`, …). Three additive levels: **1 = read, 2 = write, 3 = owner**; owner@root (the empty ltree path) owns the whole space. +- **API**: JSON-RPC 2.0 over HTTP, two endpoints: + - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `apiKey.*`). + - `/api/v1/user/rpc` — session only (an api key never authenticates here; agents can't manage agents). `whoami`, `agent.*`, `space.*`. + - Plus REST OAuth device-flow endpoints under `/api/v1/auth/*`. +- **Auth**: humans use a **session token** (OAuth device flow, GitHub/Google); agents use an **api key** (`me...`). Session + api-key secrets are sha256 (compared by equality in SQL), not argon2. +- **Embedding**: Vercel AI SDK; OpenAI `text-embedding-3-small` (1536-dim) in production; Ollama supported for local dev. +- **CLI**: `me` binary — `login`, `logout`, `whoami`, `space`, `group`, `access`, `agent`, `apikey`, `memory` (+ top-level aliases like `me search`, `me create`), `mcp`, `claude`/`codex`/`gemini`/`opencode`, `serve`, `pack`. + +## Principals, members, spaces (terminology) + +- **Principal** = the union **user | agent | group** (`principal.kind` = `'u'` | `'a'` | `'g'`). The space roster (`principal_space`) holds principals. `principal.member_id` is a generated column equal to `id` for users/agents (NOT groups). +- **Member** = the **user/agent** sense only — group members and api-key holders. So params split as `principalId` (roster / grants, any kind) vs `memberId` (group membership, api keys; u|a only). The space-roster surface is principal-centric (`principal.*` methods, `SpacePrincipal` type), reserving "member" for u|a. +- **Space**: identified by an immutable 12-char `slug` (which is the `me_` schema name, the api-key prefix, and the `X-Me-Space` value) and a renamable `name`. `me space rename` changes only the name. No org / engine / shard concepts. +- **Admin**: `principal_space.admin` is *structural* authority (manage groups + roster), distinct from data ownership (owner@root via `tree_access`). It transfers **transitively** through a group whose own `principal_space.admin` is true; agents are never admins. +- **Transitive membership** (Model 2): a group member gains the group's space membership, its space-admin (if the group is admin), and its tree-access grants. ## Project Structure ``` packages/ - cli/ # CLI and MCP server (the `me` binary) - client/ # TypeScript client for the engine API - engine/ # Core engine (database operations, search, embedding) - protocol/ # Shared types and Zod schemas (JSON-RPC methods) - hosted/ # Hosted/multi-tenant provisioning + auth/ # auth-schema store: users, sessions, oauth accounts, device flow + cli/ # CLI + MCP server (the `me` binary) + claude-plugin/# Claude Code plugin (capture hooks, slash commands) + client/ # TS client: createMemoryClient + createUserClient (+ auth device flow) + database/ # schema migrations (auth, core, space) + shared migrate kit docs-site/ # Next.js static site that renders `docs/` for docs.memory.build + embedding/ # vector embedding providers (OpenAI, Ollama) + engine/ # runtime stores over the SQL functions: core (control plane) + space (data plane) + protocol/ # shared Zod schemas + types: memory + space + user contracts; auth/fields/headers/jsonrpc/errors/version + server/ # HTTP server, routing, RPC handlers, OAuth, first-login provisioning + web/ # React UI served by `me serve` (talks to the same-origin /rpc proxy) + worker/ # background embedding queue processor packs/ # Memory packs (pre-built knowledge collections) docs/ cli/ # CLI command reference (one file per command group) mcp/ # MCP tool reference (one file per tool) ``` -> **Note**: `packages/hosted` is the target package name; the current implementation is split across `packages/accounts` (org/member/engine management, OAuth), `packages/server` (HTTP server, routing, RPC handlers), `packages/embedding` (vector embedding providers), and `packages/worker` (background embedding queue processor). - ## Build, Lint, and Test Always use the `./bun` wrapper script (auto-installs the pinned Bun version): @@ -71,6 +93,9 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): **Important**: After making code changes, always run `./bun run check`. +> `packages/web` and `packages/docs-site` are excluded from the root typecheck +> (they have their own); `./bun run check` does not cover them. + ### Database integration tests `*.integration.test.ts` files run against a real PostgreSQL 18 with the @@ -85,8 +110,8 @@ TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db To run a single integration file directly, pass `--timeout 30000` (as `test:db` does). bun's default 5s timeout isn't enough over a remote ghost connection — -the migrating `beforeAll` provisions a full core/space and overruns it, which -surfaces as a misleading "beforeEach/afterEach hook timed out": +the migrating `beforeAll` provisions a full auth/core/space and overruns it, +which surfaces as a misleading "beforeEach/afterEach hook timed out": ```bash TEST_DATABASE_URL="$(ghost connect testing_me)" \ @@ -94,26 +119,26 @@ TEST_DATABASE_URL="$(ghost connect testing_me)" \ ``` Isolation is **schema-level** (ghost forbids `create database`): each test -provisions its own schema — `core_test_` for core, `metest_` for -spaces — so the suites are fully concurrent and parallel-safe across files. -Both core and space migrations are templated, so production uses `core` / -`me_` while tests target throwaway schemas and never touch real data. -Test spaces deliberately use the `metest_` prefix (not the production `me_`) -via `migrateSpace`'s `schema` override, so leftovers are distinguishable from -real spaces by name alone. +provisions its own throwaway schema(s) — `core_test_` for core, +`auth_test_` for auth, `metest_` for spaces — so the suites are +fully concurrent and parallel-safe across files. All migrations are templated, +so production uses `core` / `auth` / `me_` while tests target throwaway +schemas and never touch real data. Test spaces deliberately use the `metest_` +prefix (not the production `me_`) so leftovers are distinguishable from real +spaces by name alone. `test:db` first runs `test:db:clean` (`scripts/clean-test-schemas.ts`) to -reclaim orphaned `core_test_*` / `metest_*` schemas left by hard-interrupted -runs. It is age-gated (only drops schemas older than 60 min, so a concurrent -`test:db` sharing the database is safe) and a no-op against a production -database — neither pattern can match a real schema. Use `test:db:clean:all` +reclaim orphaned `core_test_*` / `auth_test_*` / `metest_*` schemas left by +hard-interrupted runs. It is age-gated (only drops schemas older than 60 min, so +a concurrent `test:db` sharing the database is safe) and a no-op against a +production database — no pattern can match a real schema. Use `test:db:clean:all` for a deliberate full reset when nothing else is using the database. ## Style Guides **TypeScript**: Biome for linting and formatting. Config in `biome.json`. -**SQL**: Lowercase keywords, leading-comma table definitions, inline comments after columns, native `uuid` with `uuidv7()`. +**SQL**: Lowercase keywords, leading-comma table definitions, inline comments after columns, native `uuid` with `uuidv7()`. Logic lives in SQL functions; the TS stores call functions rather than querying tables directly. ```sql create table me.memory @@ -128,44 +153,29 @@ create table me.memory ## Key Design Decisions -- **Single table**: All memory types live in `me.memory`. Complexity comes from conventions in `meta` and `tree`, not schema proliferation. -- **Database-native**: Uses PostgreSQL extensions (ltree, pgvector, JSONB GIN, tstzrange, BM25) instead of application-layer abstractions. -- **Flexibility over prescription**: `meta` accepts any JSON, `tree` paths are user-defined, `temporal` is optional. No enforced conventions. -- **MCP compatibility**: All tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). - -## Database driver migration: Bun.SQL → postgres.js (in progress) - -**Why:** `Bun.SQL` (`new Bun.SQL(...)`) does not return a pooled connection after a -query or `begin()` callback errors — after `max` such errors the pool drains and the -next acquire hangs forever (Bun bug [oven-sh/bun#22395](https://github.com/oven-sh/bun/issues/22395), -reproduced on 1.3.13 and 1.3.14). Any *expected* constraint violation on a long-lived -pool — e.g. the engine/accounts pools in `packages/server/index.ts` — can wedge the -server until restart. Both `postgres` (postgres.js) and `pg` fix it on the Bun runtime; -we use **postgres.js** because `Bun.SQL`'s API was modeled on it, so it's a near-drop-in. - -**Done & verified (local + ghost):** the migrate path — `packages/database/core/migrate/*`, -`packages/database/space/migrate/*` (incl. `test-utils.ts`), and `scripts/migrate-db.ts`. - -**Remaining**, package by package, each behind its own integration tests: -`packages/engine` (`db.ts`, `ops/*`, `migrate/*`), `packages/accounts` (`db.ts`, `ops/*`, -`migrate/*`), `packages/server` (`index.ts` pools, `context.ts`, handlers), `packages/worker`. -Spot-check `halfvec`/`ltree`/`tstzrange` round-trips and the `sql(identifier)` interpolations. - -**Per-file recipe:** -- Add `"postgres": "^3.4.9"` to the package's `package.json`. -- `import { SQL } from "bun"` → `import postgres from "postgres"` (value) and/or - `import type { Sql as SQL } from "postgres"` (type). Type a param that receives a - transaction (`sql.begin`'s `tx`) as `ISql<{}>` — both `Sql` and `TransactionSql` extend - `ISql`; keep `Sql<{}>` only for code that calls `.begin`. -- `new Bun.SQL(url, { max, idleTimeout, maxLifetime, connectionTimeout })` → - `postgres(url, { max, idle_timeout, max_lifetime, connect_timeout, onnotice: () => {} })` - (snake_case; `onnotice` silences routine migration NOTICEs). -- `sql.close()` → `sql.end()`. -- `error instanceof SQL.PostgresError` → duck-type (`(error as { position?: unknown }).position`). -- Rows: postgres.js returns a typed `Row` (index signature), but `noUncheckedIndexedAccess` - makes `rows[0]` possibly-`undefined` → `const [row] = ...; row?.col`, and drop - `(r: { col: T })` annotations on `.map` callbacks (`r` is `Row`). - -**Test gotcha:** `expect(sql\`…\`).rejects` **hangs** in bun:test — it doesn't drive -postgres.js's lazy `PendingQuery`. Assert query failures with try/catch (see `expectReject` -in `migrate/test-utils.ts`). `expect(migrateX(…)).rejects` is fine (real async-fn Promise). +- **One DB, one pool**: `auth` + `core` + every `me_` live in one Postgres database behind one postgres.js pool (plus a dedicated worker pool). Sharding / pgdog distribution is deferred; the per-slug schema model keeps a future re-split cheap. +- **Single memory table per space**: all memory lives in `me_.memory`. Complexity comes from conventions in `meta` and `tree`, not schema proliferation. +- **Database-native**: PostgreSQL extensions (ltree, pgvector/halfvec, JSONB GIN, tstzrange, BM25) instead of application-layer abstractions. +- **Access via `tree_access`, not RLS**: RLS was unperformant. `build_tree_access` produces a `_tree_access` jsonb passed into the space functions; there is no `me.user_id` GUC. Three levels (read/write/owner); owner@root delegates within a subtree. +- **Two endpoints, two auth modes**: memory RPC (session or api key + `X-Me-Space`) vs user RPC (session only). `extractBearerToken` is the one shared auth helper. +- **Principal vs member** terminology (see above): principal = u|a|g; member/`memberId` = u|a. +- **CLI credentials**: the credentials file (`~/.config/me/credentials.yaml`, 0600) stores only the **session token + active space** per server. **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE`. *TODO*: move the session token into the OS keychain with a 0600-file fallback. +- **Header constants** (`CLIENT_VERSION_HEADER`, `SPACE_HEADER`) live in `@memory.build/protocol/headers`. +- **MCP compatibility**: all tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). + +## Database driver: postgres.js + +The runtime is fully on **postgres.js**. We moved off `Bun.SQL` because it does +not return a pooled connection after a query or `begin()` callback errors — +after `max` such errors the pool drains and the next acquire hangs forever (Bun +bug [oven-sh/bun#22395](https://github.com/oven-sh/bun/issues/22395)). The single +application pool + the worker pool + all stores and migrations use postgres.js. +The only remaining `Bun.SQL` use is `scripts/setup.ts`, a dev-only +create-database helper (short-lived, no long pool — the bug doesn't bite). + +Gotchas when writing DB code / tests: + +- Pass jsonb to SQL functions via `sql.json(v)` — a raw `JSON.stringify` double-encodes and a raw array sends as a PG array. +- `noUncheckedIndexedAccess` makes `rows[0]` possibly-`undefined` → `const [row] = ...; row?.col`; don't annotate `.map((r: {col}) => …)` (the row is a typed `Row`). +- `expect(sql\`…\`).rejects` **hangs** in bun:test — it doesn't drive postgres.js's lazy `PendingQuery`. Assert query failures with try/catch (see `expectReject` in `packages/database/migrate/test-utils.ts`). `expect(migrateX(…)).rejects` is fine (a real async-fn Promise). +- `to_bm25query(text, index_name text)` — the index name is `text`, not `regclass`. citext function params: compare with `_x::citext` or it silently degrades to case-sensitive `text = text`. diff --git a/scripts/clean-test-schemas.ts b/scripts/clean-test-schemas.ts index db5f1ed..f457535 100644 --- a/scripts/clean-test-schemas.ts +++ b/scripts/clean-test-schemas.ts @@ -11,11 +11,12 @@ // schema names that are impossible in production: // // * `core_test_*` — core tests; production's control plane is the bare `core`. +// * `auth_test_*` — auth tests; production's auth schema is the bare `auth`. // * `metest_*` — space tests; production spaces are `me_`. Tests // deliberately provision under the `metest_` prefix (see -// packages/space/migrate/test-utils.ts) so they never share -// a name with a real space, and `metest_` does not start -// with the `me_` engine prefix. +// packages/database/space/migrate/test-utils.ts) so they +// never share a name with a real space, and `metest_` does +// not start with the `me_` engine prefix. // // The result: pointed at a real database, this script is a no-op — neither // pattern can match a production schema. @@ -32,8 +33,13 @@ import postgres, { type Sql } from "postgres"; const DEFAULT_TEST_DATABASE_URL = "postgresql://postgres@127.0.0.1:5432/postgres"; -// Production-impossible test schema patterns. core_test_, metest_. -const TEST_SCHEMA_PATTERNS = ["^core_test_[a-z0-9]+$", "^metest_[a-z0-9]{12}$"]; +// Production-impossible test schema patterns. +// core_test_, auth_test_, metest_. +const TEST_SCHEMA_PATTERNS = [ + "^core_test_[a-z0-9]+$", + "^auth_test_[a-z0-9]+$", + "^metest_[a-z0-9]{12}$", +]; const SAFE_IDENTIFIER = /^[a-z_][a-z0-9_]*$/; @@ -52,8 +58,9 @@ function parseArgs(argv: string[]): Options { `Usage: ./bun scripts/clean-test-schemas.ts [--all] [--older-than-min N] [--quiet] Drops orphaned integration-test schemas from TEST_DATABASE_URL (default -${DEFAULT_TEST_DATABASE_URL}): core_test_* and metest_* schemas. Safe against -production databases (no-op there — neither pattern can match a real schema). +${DEFAULT_TEST_DATABASE_URL}): core_test_*, auth_test_*, and metest_* schemas. +Safe against production databases (no-op there — no pattern can match a real +schema). --all Ignore age; drop every matching schema. Use only when no other test run shares the database. From 6255324e91f9268c6425c609f923166c8b8e81b6 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 09:27:18 +0200 Subject: [PATCH 068/156] docs(todo): track four untracked follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add TODO entries that were noted in review but not yet tracked: - OS keychain for the CLI session token (file fallback) — was only a code comment in credentials.ts. - Refresh docs/ to the principal/space model (retired engine/org/role docs). - Deploy: coordinate the DATABASE_URL env rename + note the orphaned accounts / old me_ schemas on any non-fresh DB. - Finish + verify the `me serve` web UI (empty bundle, excluded from CI, Monaco typecheck errors, no serve->memory-RPC test). Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/TODO.md b/TODO.md index 075ecbc..d0e1509 100644 --- a/TODO.md +++ b/TODO.md @@ -151,3 +151,55 @@ form — the right convention is what's natural for users, not what ltree accept slug/shard handling, template vars). Verified: typecheck/lint clean, unit + ghost db suites pass. (bootstrap's lock moved from a hardcoded single-key id to the shared two-key derived lock.) + +## OS keychain for CLI credentials + +The CLI credentials file (`~/.config/me/credentials.yaml`, 0600) stores the +session token + active space in plaintext. (Api keys are never stored — they +come from `ME_API_KEY` only.) A code TODO marker lives in +`packages/cli/credentials.ts`. + +- [ ] Move the session token into the OS keychain (macOS `security`, Linux + `secret-tool`, Windows credential manager) with a fallback to the 0600 + file when no keychain is available (CI, headless Linux). The file would + then hold only non-secret pointers (`default_server`, `active_space`). + +## Refresh `docs/` for the principal / space model + +The `docs/` pages (getting-started, concepts, access-control, `cli/*`, `mcp/*`) +still describe the retired engine / org / role / accounts model. `CLAUDE.md` is +now the authoritative summary of the current design. + +- [ ] Rewrite the docs to the new model: principals (user | agent | group), + spaces (immutable slug / renamable name, `X-Me-Space`), 3-level + tree-access grants, session-vs-api-key auth, and the + `me space/group/access/agent/apikey/memory` command surface. Update + `docs/cli/*` (drop engine/org/invitation/user/owner/role; add + space/group/access/agent) and `docs/mcp/*`. The docs-site renders these. + +## Deploy: env rename coordination (Phase 5) + +Phase 5 renamed the server's required DB env var and removed the accounts DB. +The server throws at boot if `DATABASE_URL` is unset (no back-compat fallback, +by design). + +- [ ] With the `multiplayer` → `main` deploy: set `DATABASE_URL` (was + `ENGINE_DATABASE_URL`) in every environment; `ACCOUNTS_DATABASE_URL` is no + longer read; pool tunables renamed `ENGINE_POOL_*` → `DB_POOL_*` and + `WORKER_ENGINE_*` → `WORKER_*` / `WORKER_DB_*`. The old `accounts` schema + (and any old RLS `me_` engine schemas) are now orphaned — no + migration drops them; remove manually if a non-fresh DB ever runs this. + +## `me serve` web UI: finish + verify + +`packages/web` is the React UI for `me serve`. Its bundled assets +(`packages/cli/serve/web-assets.generated.ts`) are an empty placeholder, +`packages/web` is excluded from the root typecheck (and has pre-existing Monaco +typecheck errors), and there is no serve → `/api/v1/memory/rpc` integration +test. The `/rpc` proxy + web client are migrated (session token + `X-Me-Space`) +but unproven at runtime. + +- [ ] Build/bundle the web UI (`scripts/bundle-web-assets.ts`), fix the Monaco + typecheck errors, and add an end-to-end check that the `me serve` `/rpc` + proxy reaches the memory endpoint. Decide whether `packages/web` should be + in CI / the root typecheck. From a94cfb065ecb123660725c167706eee254059145 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 09:46:14 +0200 Subject: [PATCH 069/156] feat(memory): user-facing tree-path normalization + ~ home directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept lenient tree paths at the boundary and normalize once to canonical ltree; the store + SQL functions stay ltree-native. - packages/database/space/path.ts: normalizeTreePath (concrete paths: `/` and `.` interchangeable, runs collapse, ends trim, root -> "", strict [A-Za-z0-9_-] labels), normalizeTreeFilter (search: same but lquery/ltxtquery wildcards pass through unvalidated), homePrefix, denormalizeTreePath. - ~ home directories: a leading `~` expands to home. for the authenticated caller; reverse-mapped to `~/…` on output for the caller's own home (so the UUID rarely surfaces). - Wired server-side in the space RPC handlers (memory.ts create/batchCreate/ update/search/tree/move/deleteTree/countTree; grant.ts set/remove/list) via inputTreePath / inputTreeFilter / displayTreePath in support.ts. The server is the single chokepoint, so CLI + MCP + web get it for free. Grant authority checks run on the normalized path. Malformed input -> VALIDATION_ERROR. Unit tests for the util; an integration test proves the ~ round-trip (stored as home..notes, displayed as ~/notes), slash normalization, memory.tree under ~, and label validation. typecheck + lint + unit (657) + integration green. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 29 ++-- packages/database/space/index.ts | 9 ++ packages/database/space/path.test.ts | 107 +++++++++++++ packages/database/space/path.ts | 146 ++++++++++++++++++ packages/server/rpc/memory/grant.ts | 28 ++-- .../rpc/memory/memory.integration.test.ts | 41 +++++ packages/server/rpc/memory/memory.ts | 73 ++++++--- packages/server/rpc/memory/support.ts | 52 ++++++- 8 files changed, 434 insertions(+), 51 deletions(-) create mode 100644 packages/database/space/path.test.ts create mode 100644 packages/database/space/path.ts diff --git a/TODO.md b/TODO.md index d0e1509..bb532ed 100644 --- a/TODO.md +++ b/TODO.md @@ -124,16 +124,25 @@ store layer, the SQL functions, `provisionUser`). At the **user-facing boundary* (RPC handlers, CLI, MCP) we want lenient input normalized once to that canonical form — the right convention is what's natural for users, not what ltree accepts. -- [ ] Add a shared `normalizeTreePath(input): string` util (home: alongside the - slug helpers in `packages/database/space`, or a small `path.ts`). Rules: - split on `/[./]+/`, drop empty segments, validate each is a legal ltree - label, join with `.`. So `/foo/bar`, `foo/bar`, `foo.bar` → `foo.bar`; and - `""`, `/`, `.` → `""` (root). Use it in **every** user-facing entry point - so they behave identically. Wire in Phase 4 with the memory/grant RPC + - CLI + MCP. -- [ ] Decide the canonical **output/display** form (echoed in search results, - `grant list`, etc.): dot-style `work.projects` (matches current docs) vs - filesystem-style `/work/projects`. Input stays lenient; output is one form. +- [x] Done — `packages/database/space/path.ts` exports `normalizeTreePath` + (strict, concrete paths), `normalizeTreeFilter` (lenient, lquery/ltxtquery + passes through), `homePrefix`, and `denormalizeTreePath`. Wired **server-side** + in the space RPC handlers (`rpc/memory/memory.ts` + `grant.ts` via + `inputTreePath`/`inputTreeFilter`/`displayTreePath` in `support.ts`), which + is the single chokepoint for CLI + MCP + web (they send raw input; the + server normalizes). Includes `~` home directories: a leading `~` expands to + `home.` (the authenticated caller), reverse- + mapped to `~/…` on output for the caller's own home. Labels allow + `[A-Za-z0-9_-]` (PG16+ hyphens). Malformed input → `VALIDATION_ERROR`. +- [ ] **Output form is only half-decided.** The caller's home reverse-maps to + slash-style `~/a/b`, but non-home paths still display dot-style + `work.projects`. Decide whether to unify all output on one separator + (dot vs slash) — if slash, update the docs (which use dot) and the + `denormalizeTreePath` non-home branch. +- [ ] Reverse-mapping only covers the **caller's own** home (other principals' + homes show the raw `home..…`). Fine for now; revisit if listing + other members' home paths becomes common (would need a uuid→`~user` or + handle lookup). ## Consolidate the migration runner logic diff --git a/packages/database/space/index.ts b/packages/database/space/index.ts index f1ed765..5372852 100644 --- a/packages/database/space/index.ts +++ b/packages/database/space/index.ts @@ -4,6 +4,15 @@ export { migrateSpace, provisionSpace, } from "./migrate/migrate"; +export { + denormalizeTreePath, + HOME_NAMESPACE, + homePrefix, + normalizeTreeFilter, + normalizeTreePath, + TreePathError, + type TreePathOptions, +} from "./path"; export { generateSlug, isValidSlug, diff --git a/packages/database/space/path.test.ts b/packages/database/space/path.test.ts new file mode 100644 index 0000000..910bf68 --- /dev/null +++ b/packages/database/space/path.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from "bun:test"; +import { + denormalizeTreePath, + homePrefix, + normalizeTreeFilter, + normalizeTreePath, + TreePathError, +} from "./path"; + +const ID = "0199c2a4-f8e1-7b3c-9d2e-5a6f08b4c1d7"; +const HOME = "home.0199c2a4f8e17b3c9d2e5a6f08b4c1d7"; + +describe("normalizeTreePath", () => { + test("root forms collapse to the empty path", () => { + for (const root of ["", "/", ".", "///", "..", "/.//"]) { + expect(normalizeTreePath(root)).toBe(""); + } + }); + + test("slash and dot separators are interchangeable; runs collapse; ends trim", () => { + expect(normalizeTreePath("foo")).toBe("foo"); + expect(normalizeTreePath("foo/bar")).toBe("foo.bar"); + expect(normalizeTreePath("foo.bar")).toBe("foo.bar"); + expect(normalizeTreePath("/foo/bar/")).toBe("foo.bar"); + expect(normalizeTreePath("foo//bar")).toBe("foo.bar"); + expect(normalizeTreePath("a/b.c")).toBe("a.b.c"); + }); + + test("hyphen and underscore labels are valid", () => { + expect(normalizeTreePath("my-project/notes_2")).toBe("my-project.notes_2"); + }); + + test("rejects illegal label characters", () => { + expect(() => normalizeTreePath("foo bar")).toThrow(TreePathError); + expect(() => normalizeTreePath("foo@bar")).toThrow(TreePathError); + expect(() => normalizeTreePath("a/b!c")).toThrow(TreePathError); + }); + + test("leading ~ expands to the caller's home", () => { + expect(normalizeTreePath("~", { home: ID })).toBe(HOME); + expect(normalizeTreePath("~/bar", { home: ID })).toBe(`${HOME}.bar`); + expect(normalizeTreePath("~.bar", { home: ID })).toBe(`${HOME}.bar`); + expect(normalizeTreePath("~/a/b", { home: ID })).toBe(`${HOME}.a.b`); + }); + + test("~ requires a home and is only valid as the first segment", () => { + expect(() => normalizeTreePath("~/bar")).toThrow(TreePathError); + expect(() => normalizeTreePath("foo/~/bar", { home: ID })).toThrow( + TreePathError, + ); + }); + + test("a literal 'home' path is not special (only ~ injects the id)", () => { + expect(normalizeTreePath("home/bar", { home: ID })).toBe("home.bar"); + }); +}); + +describe("normalizeTreeFilter", () => { + test("passes lquery / ltxtquery syntax through unvalidated", () => { + expect(normalizeTreeFilter("*")).toBe("*"); + expect(normalizeTreeFilter("*.api.*")).toBe("*.api.*"); + expect(normalizeTreeFilter("foo & bar")).toBe("foo & bar"); + }); + + test("normalizes separators and trims", () => { + expect(normalizeTreeFilter("")).toBe(""); + expect(normalizeTreeFilter("/foo/bar/")).toBe("foo.bar"); + expect(normalizeTreeFilter("foo//bar")).toBe("foo.bar"); + }); + + test("expands a leading ~ but keeps the wildcard remainder", () => { + expect(normalizeTreeFilter("~", { home: ID })).toBe(HOME); + expect(normalizeTreeFilter("~/proj.*", { home: ID })).toBe( + `${HOME}.proj.*`, + ); + expect(normalizeTreeFilter("~.*", { home: ID })).toBe(`${HOME}.*`); + }); +}); + +describe("homePrefix", () => { + test("strips hyphens from the principal id", () => { + expect(homePrefix(ID)).toBe(HOME); + }); +}); + +describe("denormalizeTreePath", () => { + test("reverse-maps the caller's home to ~ with slash separators", () => { + expect(denormalizeTreePath(HOME, { home: ID })).toBe("~"); + expect(denormalizeTreePath(`${HOME}.bar`, { home: ID })).toBe("~/bar"); + expect(denormalizeTreePath(`${HOME}.a.b`, { home: ID })).toBe("~/a/b"); + }); + + test("leaves non-home paths (and other principals' homes) unchanged", () => { + expect(denormalizeTreePath("work.projects", { home: ID })).toBe( + "work.projects", + ); + expect(denormalizeTreePath("home.deadbeef.x", { home: ID })).toBe( + "home.deadbeef.x", + ); + expect(denormalizeTreePath(`${HOME}.bar`)).toBe(`${HOME}.bar`); // no home opt + }); + + test("round-trips with normalizeTreePath", () => { + const display = denormalizeTreePath(`${HOME}.a.b`, { home: ID }); // ~/a/b + expect(normalizeTreePath(display, { home: ID })).toBe(`${HOME}.a.b`); + }); +}); diff --git a/packages/database/space/path.ts b/packages/database/space/path.ts new file mode 100644 index 0000000..462adde --- /dev/null +++ b/packages/database/space/path.ts @@ -0,0 +1,146 @@ +/** + * User-facing tree-path normalization. + * + * Memories live under an ltree `tree` path (dot-separated; the root is the empty + * path). At the user-facing boundary (RPC handlers, CLI, MCP) we accept lenient + * input and normalize it once to the canonical ltree form. The store layer and + * SQL functions stay ltree-native and only ever see canonical paths. + * + * Conventions: + * - **Separators**: `/` and `.` are interchangeable; runs collapse; leading and + * trailing separators are dropped. `/a/b`, `a/b`, `a.b` → `a.b`. + * - **Root**: ``, `/`, `.` → `` (the empty ltree path). + * - **Home**: a leading `~` segment expands to `home.`, where + * `` is the caller's principal id with hyphens stripped (a valid + * ltree label). `~` → `home.`, `~/bar` → `home..bar`. + * `~` is only meaningful as the first segment. + * - **Labels** (concrete paths): each segment must be a legal ltree label + * (`[A-Za-z0-9_-]+`, PG16+); anything else throws `TreePathError`. + * + * Two entry points: + * - `normalizeTreePath` — a concrete path (create/update/move/grant/…). Strict + * label validation. + * - `normalizeTreeFilter` — a search filter, which may be an ltree `lquery` + * (`*.api.*`) or `ltxtquery`. Expands `~` and slashes but does NOT validate + * labels, so wildcard/query syntax passes through untouched. + */ + +/** The reserved top-level namespace for per-principal home directories. */ +export const HOME_NAMESPACE = "home"; + +/** A legal ltree label (PostgreSQL 16+): letters, digits, underscore, hyphen. */ +const LTREE_LABEL = /^[A-Za-z0-9_-]+$/; + +/** Thrown on malformed user input (mapped to a validation error at the boundary). */ +export class TreePathError extends Error { + constructor(message: string) { + super(message); + this.name = "TreePathError"; + } +} + +export interface TreePathOptions { + /** + * The principal id whose home a leading `~` expands to. Required to use `~`; + * omitting it makes a `~` segment an error. + */ + home?: string; +} + +/** The canonical ltree prefix for a principal's home: `home.`. */ +export function homePrefix(principalId: string): string { + const id = principalId.replace(/-/g, ""); + if (!LTREE_LABEL.test(id)) { + throw new TreePathError( + `invalid home principal id: ${JSON.stringify(principalId)}`, + ); + } + return `${HOME_NAMESPACE}.${id}`; +} + +/** Split on runs of `/` or `.`, dropping empty segments. */ +function splitSegments(input: string): string[] { + return input.split(/[/.]+/).filter((s) => s.length > 0); +} + +/** + * Normalize a concrete tree path to canonical ltree. Lenient on separators and + * a leading `~`; strict on labels. Returns `""` for the root. + */ +export function normalizeTreePath( + input: string, + opts: TreePathOptions = {}, +): string { + const segments = splitSegments(input); + if (segments.length === 0) return ""; + + const out: string[] = []; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i] as string; + if (seg === "~") { + if (i !== 0) { + throw new TreePathError("'~' is only valid as the first path segment"); + } + if (opts.home === undefined) { + throw new TreePathError("'~' (home) is not available here"); + } + out.push(homePrefix(opts.home)); // already a valid `home.` ltree + continue; + } + if (!LTREE_LABEL.test(seg)) { + throw new TreePathError( + `invalid tree path segment: ${JSON.stringify(seg)}`, + ); + } + out.push(seg); + } + return out.join("."); +} + +/** + * Normalize a search filter (lquery / ltxtquery): expand a leading `~`, treat + * `/` as a separator, collapse and trim separators — but pass wildcard/query + * syntax through unvalidated. Returns `""` when there is no filter. + */ +export function normalizeTreeFilter( + input: string, + opts: TreePathOptions = {}, +): string { + let s = input.trim(); + if (s === "") return ""; + + // Leading `~` home expansion (only as the first segment). + if (s === "~" || s.startsWith("~/") || s.startsWith("~.")) { + if (opts.home === undefined) { + throw new TreePathError("'~' (home) is not available here"); + } + s = homePrefix(opts.home) + s.slice(1); // "~/foo" → "home./foo" + } + + // Slash → dot, collapse separator runs, trim ends. + s = s + .replace(/\//g, ".") + .replace(/\.{2,}/g, ".") + .replace(/^\.+|\.+$/g, ""); + return s; +} + +/** + * Reverse of the home expansion, for display. A path under the given + * principal's home is shown with a leading `~` and slash separators + * (`home.` → `~`, `home..a.b` → `~/a/b`); everything else (including + * other principals' homes) is returned unchanged. + */ +export function denormalizeTreePath( + path: string, + opts: TreePathOptions = {}, +): string { + if (opts.home === undefined) return path; + const prefix = homePrefix(opts.home); + if (path === prefix) return "~"; + if (path.startsWith(`${prefix}.`)) { + const rest = path.slice(prefix.length + 1); + return `~/${rest.split(".").join("/")}`; + } + return path; +} diff --git a/packages/server/rpc/memory/grant.ts b/packages/server/rpc/memory/grant.ts index c044363..60fe517 100644 --- a/packages/server/rpc/memory/grant.ts +++ b/packages/server/rpc/memory/grant.ts @@ -20,6 +20,7 @@ import type { HandlerContext } from "../types"; import { callerOwnsAgent, guardCore, + inputTreePath, isSpaceManager, ownsTreePath, requireSpaceManager, @@ -50,12 +51,13 @@ async function grantSet( ): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; - await requireGrantAuthority(ctx, params.principalId, params.treePath); + const treePath = inputTreePath(ctx, params.treePath); + await requireGrantAuthority(ctx, params.principalId, treePath); await guardCore(() => ctx.core.grantTreeAccess( ctx.space.id, params.principalId, - params.treePath, + treePath, params.access, ), ); @@ -68,13 +70,10 @@ async function grantRemove( ): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; - await requireGrantAuthority(ctx, params.principalId, params.treePath); + const treePath = inputTreePath(ctx, params.treePath); + await requireGrantAuthority(ctx, params.principalId, treePath); const removed = await guardCore(() => - ctx.core.removeTreeAccessGrant( - ctx.space.id, - params.principalId, - params.treePath, - ), + ctx.core.removeTreeAccessGrant(ctx.space.id, params.principalId, treePath), ); return { removed }; } @@ -85,25 +84,26 @@ async function grantList( ): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; + const treePath = + params.treePath !== undefined && params.treePath !== null + ? inputTreePath(ctx, params.treePath) + : undefined; // Authorized when listing your OWN agent's grants, or a subtree you own, or // (broadly) as a space manager. const ownAgent = params.principalId !== undefined && params.principalId !== null && (await callerOwnsAgent(ctx, params.principalId)); - const pathOwner = - params.treePath !== undefined && - params.treePath !== null && - ownsTreePath(ctx, params.treePath); + const pathOwner = treePath !== undefined && ownsTreePath(ctx, treePath); if (!ownAgent && !pathOwner) { requireSpaceManager(ctx); } const grants = await ctx.core.listTreeAccessGrants( ctx.space.id, params.principalId ?? undefined, - params.treePath ?? undefined, + treePath, ); - return { grants: grants.map(toTreeGrantResponse) }; + return { grants: grants.map((g) => toTreeGrantResponse(g, ctx)) }; } export const grantMethods = buildRegistry() diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index a0d4814..66cb1f4 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -114,6 +114,47 @@ beforeEach(async () => { principalId = r.userId; }); +test("~ home + lenient separators normalize on input, reverse-map on output", async () => { + const home = `home.${principalId.replace(/-/g, "")}`; + + // `~/notes` is stored under the caller's home and reads back as `~/notes`. + const a = await call<{ id: string; tree: string }>("memory.create", { + content: "home note", + tree: "~/notes", + }); + expect(a.tree).toBe("~/notes"); + expect((await call<{ tree: string }>("memory.get", { id: a.id })).tree).toBe( + "~/notes", + ); + + // The raw stored ltree is home..notes — proves real expansion, not just display. + const [row] = await sql.unsafe( + `select tree::text as tree from me_${space.slug}.memory where id = $1`, + [a.id], + ); + expect(row?.tree).toBe(`${home}.notes`); + + // Slash input normalizes to dot ltree; non-home paths display canonically. + const b = await call<{ tree: string }>("memory.create", { + content: "slashy", + tree: "/work/projects/", + }); + expect(b.tree).toBe("work.projects"); + + // memory.tree with a `~` base finds the home node, reverse-mapped to `~/notes`. + const tree = await call<{ nodes: { path: string; count: number }[] }>( + "memory.tree", + { tree: "~" }, + ); + expect(tree.nodes.some((n) => n.path === "~/notes")).toBe(true); + + // An illegal label is a validation error. + await expectAppError( + call("memory.create", { content: "x", tree: "bad label" }), + "VALIDATION_ERROR", + ); +}); + test("create → get round-trips content/tree/meta and createdBy is null", async () => { const created = await call<{ id: string; createdBy: string | null }>( "memory.create", diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 2218404..1f4b99d 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -48,6 +48,7 @@ import { import { AppError } from "../errors"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; +import { displayTreePath, inputTreeFilter, inputTreePath } from "./support"; import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; // ============================================================================= @@ -120,12 +121,15 @@ function nlevel(path: string): number { return path === "" ? 0 : path.split(".").length; } -function toMemoryResponse(m: SpaceMemory): MemoryResponse { +function toMemoryResponse( + m: SpaceMemory, + ctx: SpaceRpcContext, +): MemoryResponse { return { id: m.id, content: m.content, meta: m.meta, - tree: m.tree, + tree: displayTreePath(ctx, m.tree), temporal: parseTemporal(m.temporal), hasEmbedding: m.hasEmbedding, createdAt: m.createdAt.toISOString(), @@ -168,14 +172,15 @@ async function memoryCreate( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; const id = await guard(() => store.createMemory(treeAccess, { id: params.id ?? undefined, content: params.content, meta: params.meta ?? undefined, - tree: params.tree ?? ROOT_PATH, + tree: inputTreePath(ctx, params.tree ?? ROOT_PATH), temporal: formatTemporal(params.temporal), }), ); @@ -183,7 +188,7 @@ async function memoryCreate( if (!memory) { throw new AppError("INTERNAL_ERROR", "Created memory could not be read"); } - return toMemoryResponse(memory); + return toMemoryResponse(memory, ctx); } /** memory.batchCreate — atomic across the batch. */ @@ -192,7 +197,8 @@ async function memoryBatchCreate( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; const ids = await guard(() => store.withTransaction(async (tx) => { @@ -203,7 +209,7 @@ async function memoryBatchCreate( id: m.id ?? undefined, content: m.content, meta: m.meta ?? undefined, - tree: m.tree ?? ROOT_PATH, + tree: inputTreePath(ctx, m.tree ?? ROOT_PATH), temporal: formatTemporal(m.temporal), }), ); @@ -220,13 +226,14 @@ async function memoryGet( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; const memory = await guard(() => store.getMemory(treeAccess, params.id)); if (!memory) { throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); } - return toMemoryResponse(memory); + return toMemoryResponse(memory, ctx); } /** memory.update */ @@ -235,7 +242,8 @@ async function memoryUpdate( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; const patch: { content?: string; @@ -250,7 +258,7 @@ async function memoryUpdate( patch.meta = params.meta; } if (params.tree !== undefined && params.tree !== null) { - patch.tree = params.tree; + patch.tree = inputTreePath(ctx, params.tree); } if (params.temporal !== undefined) { patch.temporal = @@ -267,7 +275,7 @@ async function memoryUpdate( if (!memory) { throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); } - return toMemoryResponse(memory); + return toMemoryResponse(memory, ctx); } /** memory.delete */ @@ -291,7 +299,8 @@ async function memorySearch( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess, embeddingConfig } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess, embeddingConfig } = ctx; // Generate the query embedding for semantic search. let vec: number[] | undefined; @@ -338,7 +347,9 @@ async function memorySearch( : undefined; const filters = { - ltree: params.tree ?? undefined, + ltree: params.tree + ? inputTreeFilter(ctx, params.tree) || undefined + : undefined, metaContains: params.meta ?? undefined, regexp: params.grep ?? undefined, ...mapTemporalFilter(params.temporal), @@ -367,7 +378,7 @@ async function memorySearch( return { results: items.map((item) => ({ - ...toMemoryResponse(item), + ...toMemoryResponse(item, ctx), score: item.score, })), total: items.length, @@ -381,9 +392,10 @@ async function memoryTree( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; - const base = params.tree ?? ""; + const base = params.tree ? inputTreePath(ctx, params.tree) : ""; // `a.b.*` matches a.b and everything under it; `*` matches all paths. const lquery = base === "" ? "*" : `${base}.*`; const entries = await guard(() => store.listTree(treeAccess, lquery)); @@ -399,7 +411,7 @@ async function memoryTree( } return true; }) - .map((e) => ({ path: e.tree, count: e.count })); + .map((e) => ({ path: displayTreePath(ctx, e.tree), count: e.count })); return { nodes }; } @@ -410,13 +422,14 @@ async function memoryMove( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; const count = await guard(() => store.moveTree( treeAccess, - params.source, - params.destination, + inputTreePath(ctx, params.source), + inputTreePath(ctx, params.destination), params.dryRun ?? false, ), ); @@ -429,10 +442,15 @@ async function memoryDeleteTree( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; const count = await guard(() => - store.deleteTree(treeAccess, params.tree, params.dryRun ?? false), + store.deleteTree( + treeAccess, + inputTreePath(ctx, params.tree), + params.dryRun ?? false, + ), ); return { count }; } @@ -443,10 +461,15 @@ async function memoryCountTree( context: HandlerContext, ): Promise { assertSpaceRpcContext(context); - const { store, treeAccess } = context as SpaceRpcContext; + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; const count = await guard(() => - store.countTree(treeAccess, { tree: params.tree }, ACCESS.read), + store.countTree( + treeAccess, + { tree: inputTreePath(ctx, params.tree) }, + ACCESS.read, + ), ); return { count }; } diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts index 5f98a40..a2edbd4 100644 --- a/packages/server/rpc/memory/support.ts +++ b/packages/server/rpc/memory/support.ts @@ -4,6 +4,12 @@ * serializers. */ +import { + denormalizeTreePath, + normalizeTreeFilter, + normalizeTreePath, + TreePathError, +} from "@memory.build/database"; import type { ApiKeyInfo, Group, @@ -29,6 +35,45 @@ import type { SpaceRpcContext } from "./types"; export { guardCore }; +// ============================================================================= +// Tree-path normalization at the user-facing boundary +// ============================================================================= + +/** + * Normalize a concrete tree path from the wire to canonical ltree, expanding a + * leading `~` to the caller's home. Maps malformed input to a validation error. + */ +export function inputTreePath(ctx: SpaceRpcContext, raw: string): string { + try { + return normalizeTreePath(raw, { home: ctx.principalId }); + } catch (e) { + throw asValidationError(e); + } +} + +/** Like `inputTreePath` but for a search filter (lquery/ltxtquery passes through). */ +export function inputTreeFilter(ctx: SpaceRpcContext, raw: string): string { + try { + return normalizeTreeFilter(raw, { home: ctx.principalId }); + } catch (e) { + throw asValidationError(e); + } +} + +/** Reverse the home expansion for display: the caller's home shows as `~/…`. */ +export function displayTreePath(ctx: SpaceRpcContext, stored: string): string { + return denormalizeTreePath(stored, { home: ctx.principalId }); +} + +function asValidationError(e: unknown): AppError { + if (e instanceof TreePathError) { + return new AppError("VALIDATION_ERROR", e.message); + } + return e instanceof AppError + ? e + : new AppError("VALIDATION_ERROR", "Invalid tree path"); +} + /** Owner-level grant (3) at the space root — owns the whole space. */ export function isSpaceOwner(context: SpaceRpcContext): boolean { return context.treeAccess.some( @@ -239,10 +284,13 @@ export function toGroupMembershipResponse( }; } -export function toTreeGrantResponse(g: TreeGrant): TreeGrantResponse { +export function toTreeGrantResponse( + g: TreeGrant, + ctx: SpaceRpcContext, +): TreeGrantResponse { return { principalId: g.principalId, - treePath: g.treePath, + treePath: displayTreePath(ctx, g.treePath), access: g.access, createdAt: g.createdAt.toISOString(), updatedAt: g.updatedAt?.toISOString() ?? null, From 92d02a1192884232a039125b24611d1f10634621 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 10:02:29 +0200 Subject: [PATCH 070/156] docs: start DECISIONS_FOR_REVIEW.md; record ~-home resolution A running log of decisions made during implementation that need maintainer sign-off. First entry: a leading `~` resolves to the authenticated principal's home, so an agent's `~` is its own home (home.), not its owner's. Co-Authored-By: Claude Opus 4.8 (1M context) --- DECISIONS_FOR_REVIEW.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 DECISIONS_FOR_REVIEW.md diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md new file mode 100644 index 0000000..8006337 --- /dev/null +++ b/DECISIONS_FOR_REVIEW.md @@ -0,0 +1,37 @@ +# Decisions for review + +Design/behavior decisions made during implementation that warrant a maintainer's +sign-off. Each entry records the decision, the alternative(s), why the call was +made, and how to change it. Once you've reviewed an entry, either fold it into +`CLAUDE.md` / `docs/` (ratified) or open a change (overridden), and delete it +here. + +--- + +## `~` (home) resolves to the authenticated principal — an agent gets its *own* home + +**Date:** 2026-06-05 · **Area:** tree-path normalization (`a94cfb0`) + +A leading `~` in a tree path expands to `home.` (UUID with hyphens +stripped) where the principal is **whoever the bearer token authenticates as**: +a human session → that user; an agent api key → that agent. So an agent's `~` is +`home.` — the agent's own isolated home — **not** its owner's home. + +**Alternative considered:** an agent's `~` maps to its owner's home +(`home.`), so an agent acting on a user's behalf writes into the +user's home tree. + +**Why this call:** `~` consistently means "me" (the authenticated principal) — +simplest mental model, no owner lookup, and agent homes stay isolated. `~` is +opt-in sugar; an agent that wants a shared/space-wide location just uses an +explicit path (e.g. `projects/x`) instead of `~`. + +**How to change it:** the home id is `ctx.principalId`, passed to the +normalizer/serializer in `packages/server/rpc/memory/support.ts` +(`inputTreePath` / `inputTreeFilter` / `displayTreePath`). To make an agent's `~` +resolve to its owner, resolve the owner id for agent principals there (the agent +principal has `ownerId`) and use it for both expansion and reverse-display. Note: +paths already stored under the current rule (`home..…`) would not +migrate automatically. + +**Status:** needs review. From cfefdac9378e920c7c6291865ad088d7c4b12e48 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 10:05:42 +0200 Subject: [PATCH 071/156] feat(memory): dot is the canonical tree-path separator; home displays as ~.blah Output/display decision: dot is canonical everywhere. denormalizeTreePath now renders the caller's home as `~.blah` (was `~/a/b`), matching the dot-style used for non-home paths and the docs. Slashes are still accepted on input (split on `[/.]+`) but never emitted. `~.blah` round-trips through normalizeTreePath. Updates the unit + integration test expectations and marks the output-form TODO resolved. typecheck + lint + unit + integration green. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 10 ++++------ packages/database/space/path.test.ts | 8 ++++---- packages/database/space/path.ts | 10 +++++----- packages/server/rpc/memory/memory.integration.test.ts | 11 ++++++----- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/TODO.md b/TODO.md index bb532ed..e6b6840 100644 --- a/TODO.md +++ b/TODO.md @@ -132,13 +132,11 @@ form — the right convention is what's natural for users, not what ltree accept is the single chokepoint for CLI + MCP + web (they send raw input; the server normalizes). Includes `~` home directories: a leading `~` expands to `home.` (the authenticated caller), reverse- - mapped to `~/…` on output for the caller's own home. Labels allow + mapped to `~.…` on output for the caller's own home. Labels allow `[A-Za-z0-9_-]` (PG16+ hyphens). Malformed input → `VALIDATION_ERROR`. -- [ ] **Output form is only half-decided.** The caller's home reverse-maps to - slash-style `~/a/b`, but non-home paths still display dot-style - `work.projects`. Decide whether to unify all output on one separator - (dot vs slash) — if slash, update the docs (which use dot) and the - `denormalizeTreePath` non-home branch. +- [x] Output form decided (2026-06-05): **dot is the canonical separator + everywhere** (`work.projects`, and home as `~.blah`); slashes are accepted + on input but never emitted. Docs already use dots, so no doc change needed. - [ ] Reverse-mapping only covers the **caller's own** home (other principals' homes show the raw `home..…`). Fine for now; revisit if listing other members' home paths becomes common (would need a uuid→`~user` or diff --git a/packages/database/space/path.test.ts b/packages/database/space/path.test.ts index 910bf68..9465221 100644 --- a/packages/database/space/path.test.ts +++ b/packages/database/space/path.test.ts @@ -84,10 +84,10 @@ describe("homePrefix", () => { }); describe("denormalizeTreePath", () => { - test("reverse-maps the caller's home to ~ with slash separators", () => { + test("reverse-maps the caller's home to ~ with the canonical dot separator", () => { expect(denormalizeTreePath(HOME, { home: ID })).toBe("~"); - expect(denormalizeTreePath(`${HOME}.bar`, { home: ID })).toBe("~/bar"); - expect(denormalizeTreePath(`${HOME}.a.b`, { home: ID })).toBe("~/a/b"); + expect(denormalizeTreePath(`${HOME}.bar`, { home: ID })).toBe("~.bar"); + expect(denormalizeTreePath(`${HOME}.a.b`, { home: ID })).toBe("~.a.b"); }); test("leaves non-home paths (and other principals' homes) unchanged", () => { @@ -101,7 +101,7 @@ describe("denormalizeTreePath", () => { }); test("round-trips with normalizeTreePath", () => { - const display = denormalizeTreePath(`${HOME}.a.b`, { home: ID }); // ~/a/b + const display = denormalizeTreePath(`${HOME}.a.b`, { home: ID }); // ~.a.b expect(normalizeTreePath(display, { home: ID })).toBe(`${HOME}.a.b`); }); }); diff --git a/packages/database/space/path.ts b/packages/database/space/path.ts index 462adde..c9fcc72 100644 --- a/packages/database/space/path.ts +++ b/packages/database/space/path.ts @@ -127,9 +127,10 @@ export function normalizeTreeFilter( /** * Reverse of the home expansion, for display. A path under the given - * principal's home is shown with a leading `~` and slash separators - * (`home.` → `~`, `home..a.b` → `~/a/b`); everything else (including - * other principals' homes) is returned unchanged. + * principal's home is shown with a leading `~`, keeping the canonical dot + * separator (`home.` → `~`, `home..a.b` → `~.a.b`); everything else + * (including other principals' homes) is returned unchanged. Dot is the + * canonical output separator throughout. */ export function denormalizeTreePath( path: string, @@ -139,8 +140,7 @@ export function denormalizeTreePath( const prefix = homePrefix(opts.home); if (path === prefix) return "~"; if (path.startsWith(`${prefix}.`)) { - const rest = path.slice(prefix.length + 1); - return `~/${rest.split(".").join("/")}`; + return `~${path.slice(prefix.length)}`; // home..a.b → ~.a.b } return path; } diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index 66cb1f4..82d523d 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -117,14 +117,15 @@ beforeEach(async () => { test("~ home + lenient separators normalize on input, reverse-map on output", async () => { const home = `home.${principalId.replace(/-/g, "")}`; - // `~/notes` is stored under the caller's home and reads back as `~/notes`. + // `~/notes` (slash accepted on input) stores under the caller's home and + // reads back in canonical dot form as `~.notes`. const a = await call<{ id: string; tree: string }>("memory.create", { content: "home note", tree: "~/notes", }); - expect(a.tree).toBe("~/notes"); + expect(a.tree).toBe("~.notes"); expect((await call<{ tree: string }>("memory.get", { id: a.id })).tree).toBe( - "~/notes", + "~.notes", ); // The raw stored ltree is home..notes — proves real expansion, not just display. @@ -141,12 +142,12 @@ test("~ home + lenient separators normalize on input, reverse-map on output", as }); expect(b.tree).toBe("work.projects"); - // memory.tree with a `~` base finds the home node, reverse-mapped to `~/notes`. + // memory.tree with a `~` base finds the home node, reverse-mapped to `~.notes`. const tree = await call<{ nodes: { path: string; count: number }[] }>( "memory.tree", { tree: "~" }, ); - expect(tree.nodes.some((n) => n.path === "~/notes")).toBe(true); + expect(tree.nodes.some((n) => n.path === "~.notes")).toBe(true); // An illegal label is a validation error. await expectAppError( From d3bfa25ee2d9a613c58c67064dd9323575784bdb Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 10:11:53 +0200 Subject: [PATCH 072/156] feat(worker): move embedding write-back into space SQL functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker held the last inline DML in the new model. Add three space functions to space/migrate/idempotent/003_embedding_queue.sql: - complete_embedding(queue_id, memory_id, embedding_version, embedding): version-guarded memory write + atomic queue finalization (completed when written, cancelled when the memory was superseded/deleted between claim and embed); returns the outcome. - fail_embedding(queue_id, error): record a transient error, leave outcome NULL so the row retries; no-op once terminal. - release_embedding(queue_id): undo the claim's attempt increment for rate limits; floors at 0, no-op once terminal. process.ts calls them via tx.unsafe (like the existing claim_embedding_batch / prune_embedding_queue) and now holds zero inline UPDATE/INSERT/DELETE — finishing the "logic in DB functions, TS calls functions" cutover for the worker. The existing process integration tests regression-guard behavior; new tests cover the functions directly, including the write-back-time version mismatch → cancelled (not reachable via the claim-time paths) and fail/release no-op on terminal rows. typecheck + lint + unit (670) + integration (161) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 18 ++-- .../idempotent/003_embedding_queue.sql | 82 ++++++++++++++++ packages/worker/process.integration.test.ts | 95 +++++++++++++++++++ packages/worker/process.ts | 66 +++++-------- 4 files changed, 211 insertions(+), 50 deletions(-) diff --git a/TODO.md b/TODO.md index e6b6840..b401124 100644 --- a/TODO.md +++ b/TODO.md @@ -59,14 +59,16 @@ raw `UPDATE embedding_queue …` / `UPDATE memory SET embedding …` statements, against the "logic in DB functions, TS calls functions" principle the rest of the cutover follows. -- [ ] Add space SQL functions for the write-back path (e.g. - `complete_embedding(queue_id, memory_id, embedding_version, embedding)` - that does the version-guarded memory update + sets the queue outcome to - `completed`/`cancelled` atomically, plus `fail_embedding(queue_id, error)` - and the rate-limit `release_embedding(queue_id)` attempt-undo), and have - `process.ts` call those instead of inline SQL. Claim already goes through - `claim_embedding_batch`; this finishes the job for the write-back/prune - side so the worker holds no embedded SQL. +- [x] Done (2026-06-05) — added `complete_embedding(queue_id, memory_id, + embedding_version, embedding)` (version-guarded memory write + atomic + `completed`/`cancelled` queue finalization, returns the outcome), + `fail_embedding(queue_id, error)` (record transient error, leave outcome + NULL), and `release_embedding(queue_id)` (attempt-undo for rate limits) to + `space/migrate/idempotent/003_embedding_queue.sql`. `process.ts` calls them + via `tx.unsafe` (like the existing claim/prune); it now holds zero inline + DML. Existing process integration tests regression-guard the behavior; new + tests cover the functions directly (incl. the write-back-time version + mismatch → `cancelled`, and fail/release no-op once terminal). ## Decision: `core` and `space` are one package (`@memory.build/database`) diff --git a/packages/database/space/migrate/idempotent/003_embedding_queue.sql b/packages/database/space/migrate/idempotent/003_embedding_queue.sql index 5aba816..08296cf 100644 --- a/packages/database/space/migrate/idempotent/003_embedding_queue.sql +++ b/packages/database/space/migrate/idempotent/003_embedding_queue.sql @@ -161,3 +161,85 @@ $func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; + +------------------------------------------------------------------------------- +-- write-back: complete_embedding / fail_embedding / release_embedding +-- The worker claims with claim_embedding_batch, generates embeddings out of +-- band, then finalizes each row through one of these (so the worker holds no +-- inline SQL). Claim and write-back are separate transactions; on a transient +-- failure the row keeps outcome NULL and becomes claimable again after its +-- visibility timeout. +------------------------------------------------------------------------------- + +-- Version-guarded write-back. Writes the embedding to the memory only if its +-- embedding_version still matches the claimed version, then finalizes the queue +-- row: 'completed' when written, 'cancelled' when the memory was superseded +-- (content changed → newer version) or deleted in the meantime. Atomic; returns +-- the outcome. +create or replace function {{schema}}.complete_embedding +( _queue_id bigint +, _memory_id uuid +, _embedding_version int +, _embedding halfvec +) +returns text +as $func$ +declare + _updated int; + _outcome text; +begin + update {{schema}}.memory + set embedding = _embedding + where id = _memory_id + and embedding_version = _embedding_version + ; + get diagnostics _updated = row_count; + + _outcome = case when _updated > 0 then 'completed' else 'cancelled' end; + + update {{schema}}.embedding_queue + set outcome = _outcome + where id = _queue_id + ; + return _outcome; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +-- Record a transient embedding error without finalizing: leaves outcome NULL so +-- the row retries (the claim sweep fails it once attempts are exhausted). No-op +-- when the row is already terminal or was CASCADE-deleted with its memory. +create or replace function {{schema}}.fail_embedding +( _queue_id bigint +, _error text +) +returns void +as $func$ + update {{schema}}.embedding_queue + set last_error = _error + where id = _queue_id + and outcome is null + ; +$func$ +language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +-- Undo the attempt increment from claim (rate-limit backoff): a transient rate +-- limit must not consume the attempt budget. No-op once the row is terminal. +create or replace function {{schema}}.release_embedding +( _queue_id bigint +) +returns void +as $func$ + update {{schema}}.embedding_queue + set attempts = greatest(attempts - 1, 0) + where id = _queue_id + and outcome is null + ; +$func$ +language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; diff --git a/packages/worker/process.integration.test.ts b/packages/worker/process.integration.test.ts index 8672457..f23dd4c 100644 --- a/packages/worker/process.integration.test.ts +++ b/packages/worker/process.integration.test.ts @@ -356,3 +356,98 @@ describe("processBatch integration (space model)", () => { expect(result).toEqual({ claimed: 0, succeeded: 0, failed: 0 }); }); }); + +describe("write-back SQL functions", () => { + beforeEach(clearPending); + + const zeroVec = `[${Array.from({ length: 1536 }, () => 0).join(",")}]`; + + async function pendingRow(content: string) { + const memoryId = await insertMemory(content); + const [q] = await getQueueEntries(memoryId); + return { + memoryId, + queueId: q?.id as string, + version: Number(q?.embedding_version), + }; + } + + test("complete_embedding writes the vector and marks 'completed' on a version match", async () => { + const { memoryId, queueId, version } = await pendingRow("complete me"); + + const [r] = (await sql.unsafe( + `SELECT ${schema}.complete_embedding($1, $2, $3, $4::halfvec) AS outcome`, + [queueId, memoryId, version, zeroVec], + )) as { outcome: string }[]; + expect(r?.outcome).toBe("completed"); + + const [mem] = await sql.unsafe( + `SELECT embedding FROM ${schema}.memory WHERE id = $1`, + [memoryId], + ); + expect(mem?.embedding).not.toBeNull(); + const [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBe("completed"); + }); + + test("complete_embedding cancels (no write) when the version no longer matches", async () => { + const { memoryId, queueId, version } = await pendingRow("superseded"); + + const [r] = (await sql.unsafe( + `SELECT ${schema}.complete_embedding($1, $2, $3, $4::halfvec) AS outcome`, + [queueId, memoryId, version + 1, zeroVec], // stale version + )) as { outcome: string }[]; + expect(r?.outcome).toBe("cancelled"); + + const [mem] = await sql.unsafe( + `SELECT embedding FROM ${schema}.memory WHERE id = $1`, + [memoryId], + ); + expect(mem?.embedding).toBeNull(); // not written + const [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBe("cancelled"); + }); + + test("fail_embedding records last_error and leaves outcome NULL; no-op once terminal", async () => { + const { memoryId, queueId } = await pendingRow("fail me"); + + await sql.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + queueId, + "boom", + ]); + let [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBeNull(); + expect(q?.last_error).toBe("boom"); + + // Finalize, then a later fail must not touch the terminal row. + await sql.unsafe( + `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE id = $1`, + [queueId], + ); + await sql.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + queueId, + "later", + ]); + [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBe("completed"); + expect(q?.last_error).toBe("boom"); // unchanged + }); + + test("release_embedding decrements attempts, floors at 0, no-op once terminal", async () => { + const { memoryId, queueId } = await pendingRow("release me"); + await sql.unsafe( + `UPDATE ${schema}.embedding_queue SET attempts = 2 WHERE id = $1`, + [queueId], + ); + + await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); + expect(Number((await getQueueEntries(memoryId))[0]?.attempts)).toBe(1); + + await sql.unsafe( + `UPDATE ${schema}.embedding_queue SET attempts = 0 WHERE id = $1`, + [queueId], + ); + await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); + expect(Number((await getQueueEntries(memoryId))[0]?.attempts)).toBe(0); + }); +}); diff --git a/packages/worker/process.ts b/packages/worker/process.ts index 301671d..6a80f3f 100644 --- a/packages/worker/process.ts +++ b/packages/worker/process.ts @@ -162,12 +162,9 @@ export async function processBatch( await sql.begin(async (tx) => { await prepareTx(tx as unknown as Sql, schema, timeouts); for (const row of claimed) { - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET attempts = greatest(attempts - 1, 0) - WHERE id = $1 AND outcome IS NULL`, - [row.queue_id], - ); + await tx.unsafe(`SELECT ${schema}.release_embedding($1)`, [ + row.queue_id, + ]); } }); } @@ -210,42 +207,29 @@ export async function processBatch( // exhausted. (Row may be CASCADE-deleted if the memory was // deleted; 0 rows updated is fine.) const error = result?.error ?? "No embedding result returned"; - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET last_error = $1 - WHERE id = $2 AND outcome IS NULL`, - [error, row.queue_id], - ); + await tx.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + row.queue_id, + error, + ]); failed++; return; } - // Version-guarded write to memory. + // Version-guarded write-back: writes the memory iff its version + // still matches the claim and finalizes the queue row — + // 'completed', or 'cancelled' if the memory was superseded + // (content changed → newer version) or deleted between claim + // and embed. const vecLiteral = `[${result.embedding.join(",")}]`; - const updated = await tx.unsafe( - `UPDATE ${schema}.memory - SET embedding = $1::halfvec - WHERE id = $2 AND embedding_version = $3 - RETURNING id`, - [vecLiteral, row.memory_id, row.embedding_version], + await tx.unsafe( + `SELECT ${schema}.complete_embedding($1, $2, $3, $4::halfvec)`, + [ + row.queue_id, + row.memory_id, + row.embedding_version, + vecLiteral, + ], ); - - if (updated.length === 0) { - // Content changed or memory deleted between claim and embed. - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET outcome = 'cancelled' - WHERE id = $1`, - [row.queue_id], - ); - } else { - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET outcome = 'completed' - WHERE id = $1`, - [row.queue_id], - ); - } succeeded++; }); } catch (error) { @@ -261,12 +245,10 @@ export async function processBatch( try { await sql.begin(async (tx) => { await prepareTx(tx as unknown as Sql, schema, timeouts); - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET last_error = $1 - WHERE id = $2 AND outcome IS NULL`, - [err.message, row.queue_id], - ); + await tx.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + row.queue_id, + err.message, + ]); }); } catch (recordError) { warning("Failed to record embedding row write-back error", { From 2948e2900d0d82cbf29f3033979db0bc2074a3e8 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 10:20:02 +0200 Subject: [PATCH 073/156] fix(worker): release_embedding resets vt so rate-limited rows retry promptly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit release_embedding is the inverse of a claim, but it only undid the attempt increment — not the visibility timeout. So a row released after a rate limit sat out the full claim lock (lock_duration, ~5 min) before any worker could retry it, even though the worker's own rate-limit backoff is typically seconds. (This latent behavior predates the refactor — the original inline SQL had it too.) Reset `vt = now()` alongside the attempts decrement so the row is immediately claimable again; the worker loop's backoff (which honors Retry-After) paces the actual retry. Tests assert the released row is claimable (vt <= now()) both directly and end-to-end through the 429 path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../idempotent/003_embedding_queue.sql | 9 ++++-- packages/worker/process.integration.test.ts | 31 ++++++++++++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/packages/database/space/migrate/idempotent/003_embedding_queue.sql b/packages/database/space/migrate/idempotent/003_embedding_queue.sql index 08296cf..048a083 100644 --- a/packages/database/space/migrate/idempotent/003_embedding_queue.sql +++ b/packages/database/space/migrate/idempotent/003_embedding_queue.sql @@ -227,8 +227,12 @@ language sql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; --- Undo the attempt increment from claim (rate-limit backoff): a transient rate --- limit must not consume the attempt budget. No-op once the row is terminal. +-- Undo a claim for a transient rate limit — the inverse of claim_embedding_batch: +-- decrement attempts (the rate limit must not consume the attempt budget) AND +-- reset the visibility timeout so the row is immediately claimable again. +-- Without resetting vt the row would sit out the full claim lock (~minutes) +-- before retrying; the worker's own rate-limit backoff (honoring Retry-After) +-- paces the actual retry. No-op once the row is terminal. create or replace function {{schema}}.release_embedding ( _queue_id bigint ) @@ -236,6 +240,7 @@ returns void as $func$ update {{schema}}.embedding_queue set attempts = greatest(attempts - 1, 0) + , vt = now() where id = _queue_id and outcome is null ; diff --git a/packages/worker/process.integration.test.ts b/packages/worker/process.integration.test.ts index f23dd4c..24bd4cd 100644 --- a/packages/worker/process.integration.test.ts +++ b/packages/worker/process.integration.test.ts @@ -208,11 +208,14 @@ describe("processBatch integration (space model)", () => { ), ).rejects.toBeInstanceOf(RateLimitError); - // claim incremented attempts to 1, the RateLimitError handler decremented - // it back to 0 so the transient failure isn't charged. + // claim incremented attempts to 1 and locked the row (vt in the future); + // the RateLimitError handler released it — attempts back to 0 (the + // transient failure isn't charged) and vt reset so it's immediately + // claimable again rather than waiting out the full claim lock. const queue = await getQueueEntries(memoryId); const entry = queue.find((q) => q.outcome === null); expect(entry?.attempts).toBe(0); + expect((entry?.vt as Date).getTime()).toBeLessThanOrEqual(Date.now()); } finally { server.stop(); } @@ -433,21 +436,39 @@ describe("write-back SQL functions", () => { expect(q?.last_error).toBe("boom"); // unchanged }); - test("release_embedding decrements attempts, floors at 0, no-op once terminal", async () => { + test("release_embedding decrements attempts, resets vt, floors at 0, no-op once terminal", async () => { const { memoryId, queueId } = await pendingRow("release me"); + // Simulate a claim: attempt charged + locked (vt pushed into the future). await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET attempts = 2 WHERE id = $1`, + `UPDATE ${schema}.embedding_queue + SET attempts = 2, vt = now() + interval '5 minutes' WHERE id = $1`, [queueId], ); await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); - expect(Number((await getQueueEntries(memoryId))[0]?.attempts)).toBe(1); + const [released] = (await sql.unsafe( + `SELECT attempts, (vt <= now()) AS claimable + FROM ${schema}.embedding_queue WHERE id = $1`, + [queueId], + )) as { attempts: number; claimable: boolean }[]; + expect(Number(released?.attempts)).toBe(1); + expect(released?.claimable).toBe(true); // vt reset → eligible again now + // Floors at 0. await sql.unsafe( `UPDATE ${schema}.embedding_queue SET attempts = 0 WHERE id = $1`, [queueId], ); await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); expect(Number((await getQueueEntries(memoryId))[0]?.attempts)).toBe(0); + + // No-op once terminal. + await sql.unsafe( + `UPDATE ${schema}.embedding_queue + SET outcome = 'completed', attempts = 5 WHERE id = $1`, + [queueId], + ); + await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); + expect(Number((await getQueueEntries(memoryId))[0]?.attempts)).toBe(5); }); }); From 5f2e1d9ead30869031ace2a6673b026a1769c2ab Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 10:23:41 +0200 Subject: [PATCH 074/156] docs(todo): batch the embedding write-back to cut DB round-trips processBatch writes back one row per transaction (~N round-trips per batch). Track making it set-based (a complete_embeddings(jsonb) batch function + batched fail/release), with a note on error isolation vs the current per-row commits. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/TODO.md b/TODO.md index b401124..917273e 100644 --- a/TODO.md +++ b/TODO.md @@ -70,6 +70,27 @@ the cutover follows. tests cover the functions directly (incl. the write-back-time version mismatch → `cancelled`, and fail/release no-op once terminal). +## Worker: batch the embedding write-back (fewer DB round-trips) + +`processBatch`'s write-back loops over each claimed row in its own `sql.begin` +transaction, calling `complete_embedding` / `fail_embedding` one row at a time — +so a batch of N claimed rows costs ~N transactions / round-trips on the +write-back side (the claim is already a single call). Over a remote DB that +per-row latency dominates a batch. + +- [ ] Make the write-back set-based: a batch SQL function (e.g. + `complete_embeddings(_rows jsonb)` taking + `[{queue_id, memory_id, embedding_version, embedding}]`, doing the + version-guarded memory updates + queue finalization in one statement-pair + and returning per-row outcomes), called once per batch instead of per row. + Do the same for the transient-fail and rate-limit `release` paths (one + call covering the whole batch). Keep the version-guard + `completed`/`cancelled` semantics (data-driven, not errors). +- [ ] Decide error isolation: one transaction for the batch is simplest but a + single poison row (e.g. a malformed vector) would fail the whole batch. + Consider a per-row fallback when the batched call errors, so one bad row + doesn't block its siblings (which today each commit independently). + ## Decision: `core` and `space` are one package (`@memory.build/database`) Resolved (2026-06): merged `packages/core` + `packages/space` into a single From 96d56542ec6baa13e08a97e14e6589d2025834f7 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 10:26:38 +0200 Subject: [PATCH 075/156] =?UTF-8?q?docs:=20defer=20space-delete=20owner=20?= =?UTF-8?q?flag=20=E2=80=94=20move=20from=20TODO=20to=20DECISIONS=5FFOR=5F?= =?UTF-8?q?REVIEW?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decision: leave space.delete / space.rename gated on space-admin (transitive through admin groups); don't add a separate owner gate until requested. Recorded in DECISIONS_FOR_REVIEW.md with the revisit trigger and the deferred alternative; removed the corresponding TODO item. Co-Authored-By: Claude Opus 4.8 (1M context) --- DECISIONS_FOR_REVIEW.md | 28 ++++++++++++++++++++++++++++ TODO.md | 13 ------------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index 8006337..13490f7 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -35,3 +35,31 @@ paths already stored under the current rule (`home..…`) would not migrate automatically. **Status:** needs review. + +--- + +## Destructive space ops (`space.delete` / `space.rename`) gated on admin — no separate owner flag + +**Date:** 2026-06-05 · **Area:** core authority model + +`space.delete` and `space.rename` are gated on **space-admin** +(`principal_space.admin`, which is transitive through admin groups). `delete` +drops the whole `me_` schema — all of the space's memories — so **any** +space-admin, including one who inherited admin via a group, can destroy +everything. + +**Decision:** leave it as-is for now. Admins can delete; we will **not** add a +distinct space-**owner** notion to protect destructive ops until someone +actually asks for it. + +**Alternative (deferred):** a separate owner gate for the truly destructive ops +— e.g. a `principal_space.owner` flag, or treating owner@root as the gate — +keeping plain admin for routine structural management (groups, members, grants). +Would also need decisions on whether owner is transitive through groups +(probably not) and how ownership transfers. + +**Revisit when:** there's a request for delete protection / "are you sure" +beyond the CLI's type-the-name confirmation, or the first report of an admin +nuking a space. At that point implement the owner gate above. + +**Status:** decided (defer); revisit on request. diff --git a/TODO.md b/TODO.md index 917273e..e51bc2a 100644 --- a/TODO.md +++ b/TODO.md @@ -3,19 +3,6 @@ Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). -## Space owner flag (protect destructive ops) - -`space.delete` (and `space.rename`) are currently gated on space-admin -(`principal_space.admin`, which is transitive through admin groups). Deleting a -space drops the whole `me_` schema (all memories), so any admin — including -one who inherited admin via a group — can destroy everything. - -- [ ] Consider a distinct space-**owner** notion (e.g. a `principal_space.owner` - flag, or treating owner@root as the gate) for the truly destructive ops - (delete, and maybe transfer-ownership), keeping admin for routine - structural management (groups, members, grants). Decide whether owner is - transitive through groups (probably not) and how ownership is transferred. - ## Reconsider: api keys for users (not just agents) Keys are currently agent-only (`apiKey.create` is gated by `requireOwnedAgent`; From 223ba6f667ca1e72683329f97642accfb88ba4a6 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 10:35:44 +0200 Subject: [PATCH 076/156] feat(cli): apikey-create hint + enforce space/core import boundary (cheap wins) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two small follow-ups from the TODO list: - me apikey create: map the server NOT_FOUND (agent not in the space) to an actionable message — "Agent '' isn't in this space yet — run 'me agent add ' first". Adds a reusable isAppErrorCode() helper to util.ts (generalized from isUnauthorized). (Auto-adding the agent was considered but skipped — silently changing space membership as a side effect of minting a key is surprising.) - biome.json: noRestrictedImports overrides keep packages/database/space free of core/ and vice versa (forbidding `**/core[/**]` / `**/space[/**]` and the package root that re-exports both), preserving the cheap space/core re-split escape hatch. Verified the rule fires on a cross-import in both directions. typecheck + lint + unit (670) green. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 18 ++++++++----- biome.json | 48 +++++++++++++++++++++++++++++++++ packages/cli/commands/apikey.ts | 12 +++++++++ packages/cli/util.ts | 27 ++++++++++--------- 4 files changed, 85 insertions(+), 20 deletions(-) diff --git a/TODO.md b/TODO.md index e51bc2a..9e8a8bd 100644 --- a/TODO.md +++ b/TODO.md @@ -22,11 +22,11 @@ implies users can mint their own keys. (`requireOwnedAgent` → NOT_FOUND otherwise). `me apikey create ` surfaces that raw NOT_FOUND, so the user has to know to run `me agent add ` first. -- [ ] Improve the UX: either pre-check membership in `me apikey create` and emit - an actionable hint ("agent X isn't in this space — run `me agent add X`"), - or offer to add it (self-service `principal.add`, which is already allowed - for your own agent) before minting the key. Map the server NOT_FOUND to the - friendlier message at minimum. +- [x] Done (2026-06-05) — `me apikey create` now maps the server `NOT_FOUND` to + an actionable message ("Agent '' isn't in this space yet — run + 'me agent add ' first"). Added a reusable `isAppErrorCode` helper to + util.ts. (Auto-adding the agent was considered but skipped — silently + changing space membership as a side effect of minting a key is surprising.) ## Space invitations @@ -87,8 +87,12 @@ sharding/distribution of spaces is off the table for now. The per-slug schema mo and the `set local pgdog.shard` code stay in the `space/` module, so re-splitting later is cheap if distribution returns. -- [ ] Keep `space/` free of `core/` imports (and vice versa) so the re-split escape - hatch stays open — worth a Biome `noRestrictedImports` rule to enforce it. +- [x] Done (2026-06-05) — Biome `noRestrictedImports` overrides in `biome.json` + forbid `packages/database/space/**` from importing core (`**/core`, + `**/core/**`, or the package root `@memory.build/database` which re-exports + it) and symmetrically forbid core from importing space, each with an + explanatory message. Verified it fires on a cross-import in both + directions. ## Consolidate duplicated test-utils diff --git a/biome.json b/biome.json index d5b5d9b..dcbe9df 100644 --- a/biome.json +++ b/biome.json @@ -56,6 +56,54 @@ } } } + }, + { + "includes": ["packages/database/space/**"], + "linter": { + "rules": { + "style": { + "noRestrictedImports": { + "level": "error", + "options": { + "patterns": [ + { + "group": [ + "**/core", + "**/core/**", + "@memory.build/database" + ], + "message": "packages/database/space must not import core/ (nor the package root, which re-exports it). Keep the data plane free of the control plane so the space/core split stays cheap to undo." + } + ] + } + } + } + } + } + }, + { + "includes": ["packages/database/core/**"], + "linter": { + "rules": { + "style": { + "noRestrictedImports": { + "level": "error", + "options": { + "patterns": [ + { + "group": [ + "**/space", + "**/space/**", + "@memory.build/database" + ], + "message": "packages/database/core must not import space/ (nor the package root, which re-exports it). Keep the control plane free of the data plane so the space/core split stays cheap to undo." + } + ] + } + } + } + } + } } ], "css": { diff --git a/packages/cli/commands/apikey.ts b/packages/cli/commands/apikey.ts index b8a0d2d..3edd3ce 100644 --- a/packages/cli/commands/apikey.ts +++ b/packages/cli/commands/apikey.ts @@ -20,6 +20,7 @@ import { buildMemoryClient, buildUserClient, handleError, + isAppErrorCode, requireSession, requireSpace, resolveAgentId, @@ -61,6 +62,17 @@ function createApiKeyCreateCommand(): Command { ); }); } catch (error) { + // apiKey.create requires the agent to already be in the space; surface + // the prerequisite instead of a bare NOT_FOUND. + if (isAppErrorCode(error, "NOT_FOUND")) { + const msg = `Agent '${agent}' isn't in this space yet — run 'me agent add ${agent}' first, then 'me apikey create' again.`; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg, code: "NOT_FOUND" }, fmt, () => {}); + } + process.exit(1); + } handleError(error, fmt, { sessionServer: creds.server }); } }); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 6a22177..1f62149 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -149,21 +149,22 @@ export async function resolveAgentId( } /** - * Detect an authentication error from the server. - * - * The server's `unauthorized()` helper sends an HTTP 401 with body - * `{"error":{"message": "...", "code": "UNAUTHORIZED"}}`. The transport wraps - * that into an RpcError. The string code lands either on `data.code` - * (`appCode`) or on `code` itself depending on which envelope path the - * response took, so we check both. + * True when `error` is a server AppError with the given string code. The code + * lands either on `data.code` (`appCode`) or on `code` itself depending on which + * envelope path the response took (RpcError types `code` as number, but the + * runtime value can be the string code when the response wasn't a strict + * JSON-RPC envelope), so we check both. */ -function isUnauthorized(error: unknown): boolean { +export function isAppErrorCode(error: unknown, code: string): boolean { if (!(error instanceof RpcError)) return false; - if (error.appCode === "UNAUTHORIZED") return true; - // The server's HTTP error envelope puts the string code on the top-level - // `code` field. RpcError types `code` as number, but the runtime value can - // be a string when the response wasn't a strict JSON-RPC envelope. - return (error.code as unknown) === "UNAUTHORIZED"; + return error.appCode === code || (error.code as unknown) === code; +} + +/** + * Detect an authentication error from the server (HTTP 401 / `UNAUTHORIZED`). + */ +function isUnauthorized(error: unknown): boolean { + return isAppErrorCode(error, "UNAUTHORIZED"); } /** From 2f69c61ef0041d120eabc7203eb6bd0b7fc56564 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 11:30:16 +0200 Subject: [PATCH 077/156] feat(core): grant owner@home at join + SHARE namespace constant (INV-1) add_principal_to_space writes a real owner@home. grant when a user joins a space (idempotent, non-clobbering). Agents are excluded: agent_tree_access clamps them to their owner, so an auto agent-home grant would be inert. Add the SHARE_NAMESPACE = "share" constant for the forthcoming shared-tree grants. Co-Authored-By: Claude Opus 4.8 --- DECISIONS_FOR_REVIEW.md | 33 +++++++++ .../migrate/idempotent/006_membership.sql | 22 +++++- .../core/migrate/migrate.integration.test.ts | 71 ++++++++++++++++++- packages/database/space/path.ts | 9 +++ packages/engine/core/core.integration.test.ts | 9 ++- packages/engine/core/db.integration.test.ts | 13 +++- .../rpc/memory/management.integration.test.ts | 3 +- 7 files changed, 149 insertions(+), 11 deletions(-) diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index 13490f7..b23988e 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -63,3 +63,36 @@ beyond the CLI's type-the-name confirmation, or the first report of an admin nuking a space. At that point implement the owner gate above. **Status:** decided (defer); revisit on request. + +--- + +## Home grant at join is for users only — agents get no auto home + +**Date:** 2026-06-05 · **Area:** membership (`add_principal_to_space`, INV-1) + +`add_principal_to_space` now writes a real `owner @ home.` grant when a +**user** joins a space (the single chokepoint every join path goes through: +provisioning, invite redemption, direct add). **Agents are deliberately excluded.** + +**Why exclude agents:** `agent_tree_access` clamps an agent's effective grants to +its owner's — an agent can never exceed what its owner can reach. A typical owner +(an invited user) holds `owner@home.` and maybe `share`, but **nothing** +over `home.`. So an auto `owner@home.` grant would be clamped to +nothing: an inert, misleading row in `tree_access` that `build_tree_access` never +returns. Users have no clamp, so their home grant is always effective. + +**Tension with the `~` decision above:** that entry frames an agent's `~` as +`home.` — its own isolated home. With agents excluded here, an agent's +`~` still *resolves* to `home.` but carries **no access by default**; the +agent can only use it if its owner explicitly grants it there (and, because of the +clamp, the owner must hold that access too). + +**How to change it (give agents real homes):** options — (a) nest agent homes +under the owner (`home..…`) so the owner's home grant covers them; or +(b) in `add_principal_to_space` for an agent, also grant the **owner** +`owner@home.` so the clamp passes (owner can then see into agent homes); +or (c) relax the clamp for the agent's own home subtree. Each needs a deliberate +call on owner visibility into agent data. The gate is `and p.kind = 'u'` in +`packages/database/core/migrate/idempotent/006_membership.sql`. + +**Status:** needs review. diff --git a/packages/database/core/migrate/idempotent/006_membership.sql b/packages/database/core/migrate/idempotent/006_membership.sql index 26da08b..9d318e8 100644 --- a/packages/database/core/migrate/idempotent/006_membership.sql +++ b/packages/database/core/migrate/idempotent/006_membership.sql @@ -1,6 +1,9 @@ ------------------------------------------------------------------------------- -- add_principal_to_space --- Adds (or updates the admin flag of) a principal's membership in a space. +-- Adds (or updates the admin flag of) a principal's membership in a space, and +-- grants a joining user owner over its home directory. The single chokepoint +-- every join path goes through (provisioning, invite redemption, direct add), +-- so a user's membership always implies home ownership. ------------------------------------------------------------------------------- create or replace function {{schema}}.add_principal_to_space ( _space_id uuid @@ -12,7 +15,22 @@ as $func$ insert into {{schema}}.principal_space (space_id, principal_id, admin) values (_space_id, _principal_id, _admin) on conflict (principal_id, space_id) do update set - admin = excluded.admin -- updated_at maintained by the before-update trigger + admin = excluded.admin; -- updated_at maintained by the before-update trigger + + -- A user owns its home directory (home., hyphens stripped); see + -- packages/database/space/path.ts homePrefix() for the matching client form. + -- Users only: an agent's effective grants are clamped to its owner's by + -- agent_tree_access, so an auto home grant would be inert (the owner has no + -- access over the agent's home); groups have no home either. Idempotent and + -- non-clobbering: an existing home grant is left untouched. + insert into {{schema}}.tree_access (space_id, principal_id, tree_path, access) + select _space_id, _principal_id + , ('home.' || replace(_principal_id::text, '-', ''))::ltree + , 3 -- owner + from {{schema}}.principal p + where p.id = _principal_id + and p.kind = 'u' + on conflict (space_id, principal_id, tree_path) do nothing $func$ language sql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; diff --git a/packages/database/core/migrate/migrate.integration.test.ts b/packages/database/core/migrate/migrate.integration.test.ts index 2c5c423..f5481cd 100644 --- a/packages/database/core/migrate/migrate.integration.test.ts +++ b/packages/database/core/migrate/migrate.integration.test.ts @@ -285,6 +285,9 @@ describe("control-plane functions", () => { return row?.id as string; } + /** A principal's canonical home path (mirrors space/path.ts homePrefix). */ + const homePath = (id: string) => `home.${id.replace(/-/g, "")}`; + type Grant = { tree_path: string; access: number }; test("create_space + create_user + grant → build_tree_access returns the search_memory jsonb shape", async () => { @@ -318,7 +321,66 @@ describe("control-plane functions", () => { [userId, spaceId], ); const ta = row?.ta as Grant[]; - expect(ta).toEqual([{ tree_path: "work.projects", access: 2 }]); + // add_principal_to_space also grants the user owner@home; the explicit + // grant adds to it. + expect(ta).toContainEqual({ tree_path: "work.projects", access: 2 }); + expect(ta).toContainEqual({ tree_path: homePath(userId), access: 3 }); + expect(ta).toHaveLength(2); + }); + }); + + test("add_principal_to_space grants owner@home to users, idempotently; not agents/groups", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Homes", + ]); + const spaceId = sp?.id as string; + + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "homer"]); + const agentId = await v7(); + await sql.unsafe(`select ${s}.create_agent($1, $2, $3)`, [ + userId, // owner + `agent_${randomSlug()}`, // name + agentId, // id + ]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + `grp_${randomSlug()}`, + ]); + const groupId = grp?.id as string; + + // add each twice to prove the home grant is idempotent + for (const id of [userId, agentId, groupId]) { + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + id, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + id, + false, + ]); + } + + const grants = async (id: string): Promise => { + const rows = await sql.unsafe( + `select tree_path::text, access from ${s}.tree_access + where space_id = $1 and principal_id = $2`, + [spaceId, id], + ); + return rows as unknown as Grant[]; + }; + // the user gets exactly one owner@home grant (not duplicated by re-add) + expect(await grants(userId)).toEqual([ + { tree_path: homePath(userId), access: 3 }, + ]); + // agents are excluded (would be clamped to the owner) and groups have no home + expect(await grants(agentId)).toEqual([]); + expect(await grants(groupId)).toEqual([]); }); }); @@ -434,7 +496,9 @@ describe("control-plane functions", () => { `select ${s}.build_tree_access($1, $2) as ta`, [userId, spaceId], ); - expect(after?.ta).toEqual([]); + // still a space member (only left the group): the group grant is gone, + // but the user keeps its own home. + expect(after?.ta).toEqual([{ tree_path: homePath(userId), access: 3 }]); // second remove is a no-op const [again] = await sql.unsafe( @@ -550,7 +614,8 @@ describe("control-plane functions", () => { `select ${s}.build_tree_access($1, $2) as ta`, [userId, spaceId], ); - expect(row?.ta).toEqual([]); + // the explicit a.b grant is gone; the user keeps its own home. + expect(row?.ta).toEqual([{ tree_path: homePath(userId), access: 3 }]); }); }); diff --git a/packages/database/space/path.ts b/packages/database/space/path.ts index c9fcc72..a9a5eaa 100644 --- a/packages/database/space/path.ts +++ b/packages/database/space/path.ts @@ -28,6 +28,15 @@ /** The reserved top-level namespace for per-principal home directories. */ export const HOME_NAMESPACE = "home"; +/** + * The reserved top-level namespace for a space's shared tree. Unlike `home`, + * this is a single shared root (not per-principal) and carries no input sugar — + * `share/x` normalizes like any other path. It exists as a named constant + * because membership/invitations grant a configurable level (read/write/owner) + * at this root; see core `redeem_space_invitations`. + */ +export const SHARE_NAMESPACE = "share"; + /** A legal ltree label (PostgreSQL 16+): letters, digits, underscore, hyphen. */ const LTREE_LABEL = /^[A-Za-z0-9_-]+$/; diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts index 2454a69..071f27f 100644 --- a/packages/engine/core/core.integration.test.ts +++ b/packages/engine/core/core.integration.test.ts @@ -164,16 +164,19 @@ test("listTreeAccessGrants returns grants; filterable by principal", async () => await core.grantTreeAccess(spaceId, userId, "a.b", ACCESS.write); await core.grantTreeAccess(spaceId, userId, "c", ACCESS.owner); + // the owner also has owner@home, granted when it joined the space (beforeEach) + const home = `home.${userId.replace(/-/g, "")}`; + const all = await core.listTreeAccessGrants(spaceId); const paths = all.map((g) => g.treePath).sort(); - expect(paths).toEqual(["a.b", "c"]); + expect(paths).toEqual([home, "a.b", "c"].sort()); expect(all.find((g) => g.treePath === "c")?.access).toBe(ACCESS.owner); const forUser = await core.listTreeAccessGrants(spaceId, userId); - expect(forUser).toHaveLength(2); + expect(forUser).toHaveLength(3); expect(await core.removeTreeAccessGrant(spaceId, userId, "a.b")).toBe(true); - expect(await core.listTreeAccessGrants(spaceId)).toHaveLength(1); + expect(await core.listTreeAccessGrants(spaceId)).toHaveLength(2); }); test("api keys: create, get, list, delete (no secret leaked)", async () => { diff --git a/packages/engine/core/db.integration.test.ts b/packages/engine/core/db.integration.test.ts index 5a27c04..f32dfc3 100644 --- a/packages/engine/core/db.integration.test.ts +++ b/packages/engine/core/db.integration.test.ts @@ -78,7 +78,13 @@ test("grant + buildTreeAccess returns the search_memory jsonb shape", async () = await db.grantTreeAccess(spaceId, userId, "work.projects", 2); const ta = await db.buildTreeAccess(userId, spaceId); - expect(ta).toEqual([{ tree_path: "work.projects", access: 2 }]); + // addPrincipalToSpace also grants the user owner@home. + expect(ta).toContainEqual({ tree_path: "work.projects", access: 2 }); + expect(ta).toContainEqual({ + tree_path: `home.${userId.replace(/-/g, "")}`, + access: 3, + }); + expect(ta).toHaveLength(2); }); test("group access flows through buildTreeAccess; removeGroupMember revokes it", async () => { @@ -98,7 +104,10 @@ test("group access flows through buildTreeAccess; removeGroupMember revokes it", }); expect(await db.removeGroupMember(spaceId, groupId, userId)).toBe(true); - expect(await db.buildTreeAccess(userId, spaceId)).toEqual([]); + // still a space member: the group grant is gone, the user keeps its home. + expect(await db.buildTreeAccess(userId, spaceId)).toEqual([ + { tree_path: `home.${userId.replace(/-/g, "")}`, access: 3 }, + ]); }); test("createApiKey + validateApiKey (good / wrong secret)", async () => { diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index 73ec6a3..b382ed9 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -278,7 +278,8 @@ test("roster/group management requires admin or owner", async () => { }); test("a space admin (without owner@root) has management authority", async () => { - // an admin member with only read access, no owner grant anywhere + // an admin member with read on a path (plus its own home from joining), but no + // owner@root — so the management authority here comes from admin, not ownership const adminMember = await makeUser(); await call("principal.add", { principalId: adminMember, admin: true }); await call("grant.set", { From a4c4dcb3ddcdcf8fd10bbb7d03a90fe6349f6a44 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 11:44:52 +0200 Subject: [PATCH 078/156] feat(core): space_invitation table + redeem/create/list/revoke (INV-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Email-keyed invitations so an invite can predate the user. A partial unique index keeps one pending invite per (space, email); accepted rows stay as history. redeem_space_invitations joins each invited space (owner@home via add_principal_to_space), grants the per-invite share level at the 'share' root, and stamps accepted_at — idempotent. Add CoreStore methods + types. Co-Authored-By: Claude Opus 4.8 --- .../core/migrate/idempotent/000_update.sql | 5 + .../migrate/idempotent/009_invitation.sql | 123 ++++++++++++++++++ .../incremental/007_space_invitation.sql | 27 ++++ .../core/migrate/migrate.integration.test.ts | 99 ++++++++++++++ packages/database/core/migrate/migrate.ts | 16 +++ packages/engine/core/core.integration.test.ts | 50 +++++++ packages/engine/core/db.ts | 80 ++++++++++++ packages/engine/core/index.ts | 2 + packages/engine/core/types.ts | 28 ++++ 9 files changed, 430 insertions(+) create mode 100644 packages/database/core/migrate/idempotent/009_invitation.sql create mode 100644 packages/database/core/migrate/incremental/007_space_invitation.sql diff --git a/packages/database/core/migrate/idempotent/000_update.sql b/packages/database/core/migrate/idempotent/000_update.sql index 16fc5c7..61d6878 100644 --- a/packages/database/core/migrate/idempotent/000_update.sql +++ b/packages/database/core/migrate/idempotent/000_update.sql @@ -33,3 +33,8 @@ create or replace trigger tree_access_before_update_trg before update on {{schema}}.tree_access for each row execute function {{schema}}.update_updated_at(); + +create or replace trigger space_invitation_before_update_trg +before update on {{schema}}.space_invitation +for each row +execute function {{schema}}.update_updated_at(); diff --git a/packages/database/core/migrate/idempotent/009_invitation.sql b/packages/database/core/migrate/idempotent/009_invitation.sql new file mode 100644 index 0000000..55868d7 --- /dev/null +++ b/packages/database/core/migrate/idempotent/009_invitation.sql @@ -0,0 +1,123 @@ +------------------------------------------------------------------------------- +-- create_space_invitation +-- Issue (or update, if one is already pending) an invitation to a space, keyed +-- by invitee email. _share_access null means no share grant; otherwise it is +-- the level (1/2/3) granted at the shared root on redemption. Returns the id. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_space_invitation +( _space_id uuid +, _email citext +, _admin bool +, _share_access int +, _invited_by uuid +) +returns uuid +as $func$ + insert into {{schema}}.space_invitation (space_id, email, admin, share_access, invited_by) + values (_space_id, _email, _admin, _share_access, _invited_by) + on conflict (space_id, email) where accepted_at is null do update set + admin = excluded.admin + , share_access = excluded.share_access + , invited_by = excluded.invited_by -- updated_at maintained by the before-update trigger + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_space_invitations +-- Pending invitations for a space (accepted ones are history), with the +-- inviter's display name when still resolvable. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_space_invitations +( _space_id uuid +) +returns table +( id uuid +, email text +, admin bool +, share_access int +, invited_by uuid +, invited_by_name text +, created_at timestamptz +) +as $func$ + select i.id, i.email::text, i.admin, i.share_access, i.invited_by, p.name::text, i.created_at + from {{schema}}.space_invitation i + left join {{schema}}.principal p on p.id = i.invited_by + where i.space_id = _space_id + and i.accepted_at is null + order by i.created_at +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- revoke_space_invitation +-- Delete a pending invitation by email. Returns true if one was removed. An +-- already-accepted invitation is not revocable here (the user is a member; +-- use remove_principal_from_space). +------------------------------------------------------------------------------- +create or replace function {{schema}}.revoke_space_invitation +( _space_id uuid +, _email citext +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.space_invitation + where space_id = _space_id + and email = _email + and accepted_at is null + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- redeem_space_invitations +-- Redeem every pending invitation for a (now-registered, verified) email: join +-- the user to each space (add_principal_to_space also grants owner@home), grant +-- access at the shared root 'share' when share_access is set, and stamp +-- accepted_at. Idempotent — a second call finds nothing pending. The user must +-- already exist as a core principal. Returns one row per space joined. +------------------------------------------------------------------------------- +create or replace function {{schema}}.redeem_space_invitations +( _user_id uuid +, _email citext +) +returns table +( space_id uuid +, slug text +, name text +, admin bool +, share_access int +) +as $func$ +declare + inv record; +begin + for inv in + select i.id, i.space_id, i.admin, i.share_access + from {{schema}}.space_invitation i + where i.email = _email + and i.accepted_at is null + for update + loop + perform {{schema}}.add_principal_to_space(inv.space_id, _user_id, inv.admin); + if inv.share_access is not null then + perform {{schema}}.grant_tree_access(inv.space_id, _user_id, 'share'::ltree, inv.share_access); + end if; + update {{schema}}.space_invitation set accepted_at = pg_catalog.now() where id = inv.id; + return query + select s.id, s.slug, s.name::text, inv.admin, inv.share_access + from {{schema}}.space s + where s.id = inv.space_id; + end loop; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/incremental/007_space_invitation.sql b/packages/database/core/migrate/incremental/007_space_invitation.sql new file mode 100644 index 0000000..0c33a89 --- /dev/null +++ b/packages/database/core/migrate/incremental/007_space_invitation.sql @@ -0,0 +1,27 @@ +------------------------------------------------------------------------------- +-- space_invitation +-- Invitations to a space, keyed by invitee email so an invite can be issued +-- before the user registers. Redeemed at login (against the user's verified +-- email) by redeem_space_invitations, which joins the space (owner@home via +-- add_principal_to_space), optionally grants access at the shared root, and +-- stamps accepted_at. A *pending* invite is one with accepted_at is null; +-- accepted rows are kept as history. +------------------------------------------------------------------------------- +create table {{schema}}.space_invitation +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, space_id uuid not null references {{schema}}.space (id) on delete cascade +, email citext not null -- invitee (the key; may not be a user yet) +, admin bool not null default false -- make the user a space admin on redemption +, share_access int check (share_access in (1, 2, 3)) -- null = no share grant; else read/write/owner at 'share' +, invited_by uuid references {{schema}}.principal (id) on delete set null -- who issued it (audit) +, created_at timestamptz not null default now() +, updated_at timestamptz -- maintained by the before-update trigger +, accepted_at timestamptz -- null = pending; set on redemption +); + +-- at most one pending invite per (space, email); accepted rows are kept as +-- history, so the uniqueness is partial. email is citext, so the dedup is +-- case-insensitive. +create unique index space_invitation_pending_uq + on {{schema}}.space_invitation (space_id, email) + where accepted_at is null; diff --git a/packages/database/core/migrate/migrate.integration.test.ts b/packages/database/core/migrate/migrate.integration.test.ts index f5481cd..c0f116d 100644 --- a/packages/database/core/migrate/migrate.integration.test.ts +++ b/packages/database/core/migrate/migrate.integration.test.ts @@ -34,6 +34,7 @@ const EXPECTED_TABLES = [ "principal", "principal_space", "space", + "space_invitation", "tree_access", "version", ]; @@ -45,6 +46,7 @@ const EXPECTED_MIGRATIONS = [ "004_group_member", "005_tree_access", "006_api_key", + "007_space_invitation", ]; const EXPECTED_FUNCTIONS = [ @@ -127,6 +129,7 @@ describe("provisioned core schema", () => { "principal_space", "group_member", "tree_access", + "space_invitation", ]) { const triggers = await listTriggers(sql, canonical.schema, table); expect(triggers).toContain(`${table}_before_update_trg`); @@ -619,6 +622,102 @@ describe("control-plane functions", () => { }); }); + test("space invitations: create (upsert) / list / redeem (join + home + share) / revoke", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Invites", + ]); + const spaceId = sp?.id as string; + + // inviter must exist as a principal (invited_by FK) + const inviterId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [ + inviterId, + "inviter@example.com", + ]); + + const email = "invitee@example.com"; + const create = (admin: boolean, share: number | null) => + sql.unsafe( + `select ${s}.create_space_invitation($1, $2, $3, $4, $5) as id`, + [spaceId, email, admin, share, inviterId], + ); + + // create (read share, not admin), then re-create promotes the SAME pending + // row to admin + owner share (upsert, not a duplicate) + const [c1] = await create(false, 1); + const inviteId = c1?.id as string; + expect(inviteId).toBeTruthy(); + const [c2] = await create(true, 3); + expect(c2?.id).toBe(inviteId); + + // list: one pending invite with the updated fields + the inviter's name + const listed = await sql.unsafe( + `select * from ${s}.list_space_invitations($1)`, + [spaceId], + ); + expect(listed).toHaveLength(1); + expect(listed[0]?.email).toBe(email); + expect(listed[0]?.admin).toBe(true); + expect(listed[0]?.share_access).toBe(3); + expect(listed[0]?.invited_by_name).toBe("inviter@example.com"); + + // the invitee registers, then redeems (email match is case-insensitive) + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, email]); + const redeemed = await sql.unsafe( + `select * from ${s}.redeem_space_invitations($1, $2)`, + [userId, "INVITEE@EXAMPLE.COM"], + ); + expect(redeemed).toHaveLength(1); + expect(redeemed[0]?.space_id).toBe(spaceId); + expect(redeemed[0]?.admin).toBe(true); + expect(redeemed[0]?.share_access).toBe(3); + + // joined as admin, with owner@home (add_principal_to_space) + owner@share + const [ps] = await sql.unsafe( + `select admin from ${s}.principal_space where space_id=$1 and principal_id=$2`, + [spaceId, userId], + ); + expect(ps?.admin).toBe(true); + const [taRow] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + const ta = taRow?.ta as Grant[]; + expect(ta).toContainEqual({ tree_path: homePath(userId), access: 3 }); + expect(ta).toContainEqual({ tree_path: "share", access: 3 }); + + // accepted: gone from the pending list, and re-redeem is a no-op + expect( + await sql.unsafe(`select * from ${s}.list_space_invitations($1)`, [ + spaceId, + ]), + ).toHaveLength(0); + expect( + await sql.unsafe( + `select * from ${s}.redeem_space_invitations($1, $2)`, + [userId, email], + ), + ).toHaveLength(0); + + // revoke: a fresh pending invite is revocable once + await create(false, null); // re-invite the same email (now allowed: prior is accepted) + const [r1] = await sql.unsafe( + `select ${s}.revoke_space_invitation($1, $2) as ok`, + [spaceId, email], + ); + expect(r1?.ok).toBe(true); + const [r2] = await sql.unsafe( + `select ${s}.revoke_space_invitation($1, $2) as ok`, + [spaceId, email], + ); + expect(r2?.ok).toBe(false); + }); + }); + test("create_api_key + validate_api_key (good, wrong-secret, expired)", async () => { await withTestCore(sql, {}, async (core) => { const s = core.schema; diff --git a/packages/database/core/migrate/migrate.ts b/packages/database/core/migrate/migrate.ts index 70a2db9..a9c3181 100644 --- a/packages/database/core/migrate/migrate.ts +++ b/packages/database/core/migrate/migrate.ts @@ -35,6 +35,9 @@ import idempotent006 from "./idempotent/006_membership.sql" with { }; import idempotent007 from "./idempotent/007_grant.sql" with { type: "text" }; import idempotent008 from "./idempotent/008_api_key.sql" with { type: "text" }; +import idempotent009 from "./idempotent/009_invitation.sql" with { + type: "text", +}; import incremental001 from "./incremental/001_space.sql" with { type: "text" }; import incremental002 from "./incremental/002_principal.sql" with { type: "text", @@ -51,6 +54,9 @@ import incremental005 from "./incremental/005_tree_access.sql" with { import incremental006 from "./incremental/006_api_key.sql" with { type: "text", }; +import incremental007 from "./incremental/007_space_invitation.sql" with { + type: "text", +}; import provisionSql from "./provision.sql" with { type: "text" }; const DIR = "packages/database/core/migrate"; @@ -82,6 +88,11 @@ const incrementals: Migration[] = [ file: "incremental/006_api_key.sql", sql: incremental006, }, + { + name: "007_space_invitation", + file: "incremental/007_space_invitation.sql", + sql: incremental007, + }, ]; const idempotents: Migration[] = [ @@ -118,6 +129,11 @@ const idempotents: Migration[] = [ file: "idempotent/008_api_key.sql", sql: idempotent008, }, + { + name: "009_invitation", + file: "idempotent/009_invitation.sql", + sql: idempotent009, + }, ]; /** diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts index 071f27f..ae4ed67 100644 --- a/packages/engine/core/core.integration.test.ts +++ b/packages/engine/core/core.integration.test.ts @@ -179,6 +179,56 @@ test("listTreeAccessGrants returns grants; filterable by principal", async () => expect(await core.listTreeAccessGrants(spaceId)).toHaveLength(2); }); +test("space invitations: create / list / redeem / revoke via the store", async () => { + // spaceId + the owner userId come from beforeEach; the owner is the inviter + const email = `invitee_${rand(8)}@example.com`; + const inviteId = await core.createSpaceInvitation(spaceId, email, { + admin: true, + shareAccess: ACCESS.write, + invitedBy: userId, + }); + expect(inviteId).toBeTruthy(); + + const pending = await core.listSpaceInvitations(spaceId); + expect(pending).toHaveLength(1); + expect(pending[0]?.email).toBe(email); + expect(pending[0]?.admin).toBe(true); + expect(pending[0]?.shareAccess).toBe(ACCESS.write); + expect(pending[0]?.invitedBy).toBe(userId); + expect(pending[0]?.invitedByName).toBe(userName); + + // the invitee registers and redeems + const inviteeId = await v7(); + await core.createUser(inviteeId, email); + const joined = await core.redeemSpaceInvitations(inviteeId, email); + expect(joined).toHaveLength(1); + expect(joined[0]?.spaceId).toBe(spaceId); + expect(joined[0]?.slug).toBeTruthy(); + expect(joined[0]?.admin).toBe(true); + expect(joined[0]?.shareAccess).toBe(ACCESS.write); + + // effective access: owner@home (from joining) + write@share + const ta = await core.buildTreeAccess(inviteeId, spaceId); + expect(ta).toContainEqual({ + tree_path: `home.${inviteeId.replace(/-/g, "")}`, + access: ACCESS.owner, + }); + expect(ta).toContainEqual({ tree_path: "share", access: ACCESS.write }); + + // accepted → no longer pending; re-redeem is a no-op + expect(await core.listSpaceInvitations(spaceId)).toHaveLength(0); + expect(await core.redeemSpaceInvitations(inviteeId, email)).toHaveLength(0); + + // a fresh invite (with no share grant) is revocable once + await core.createSpaceInvitation(spaceId, email, { + admin: false, + shareAccess: null, + invitedBy: userId, + }); + expect(await core.revokeSpaceInvitation(spaceId, email)).toBe(true); + expect(await core.revokeSpaceInvitation(spaceId, email)).toBe(false); +}); + test("api keys: create, get, list, delete (no secret leaked)", async () => { const key = await core.createApiKey(userId, "ci"); expect(key.secret).toBeTruthy(); diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts index 28a3050..cf2eaf2 100644 --- a/packages/engine/core/db.ts +++ b/packages/engine/core/db.ts @@ -11,7 +11,9 @@ import type { MemberSpace, Principal, PrincipalKind, + RedeemedInvitation, Space, + SpaceInvitation, SpacePrincipal, TreeAccess, TreeGrant, @@ -136,6 +138,35 @@ export interface CoreStore { /** Hard-delete a key (revoke ≡ delete; there is no soft-revoke state). */ deleteApiKey(id: string): Promise; + /** + * Issue (or update, if one is already pending) an invitation to a space, + * keyed by invitee email — so it can be issued before the user registers. + * `shareAccess` null means no share grant. Returns the invitation id. + */ + createSpaceInvitation( + spaceId: string, + email: string, + opts: { + admin: boolean; + shareAccess: AccessLevel | null; + invitedBy: string; + }, + ): Promise; + /** Pending invitations for a space (accepted ones are history). */ + listSpaceInvitations(spaceId: string): Promise; + /** Revoke a pending invitation by email. Returns true if one was removed. */ + revokeSpaceInvitation(spaceId: string, email: string): Promise; + /** + * Redeem all pending invitations for a (now-registered, verified) email: + * join each space (owner@home), grant share access where set, mark accepted. + * Idempotent; the user must already exist as a core principal. Returns the + * spaces joined. + */ + redeemSpaceInvitations( + userId: string, + email: string, + ): Promise; + /** Run operations atomically against the same transaction. */ withTransaction(fn: (db: CoreStore) => Promise): Promise; } @@ -443,6 +474,55 @@ export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { return Boolean(row?.ok); }, + async createSpaceInvitation(spaceId, email, opts) { + const [row] = await sql` + select ${sch}.create_space_invitation( + ${spaceId}, ${email}, ${opts.admin}, ${opts.shareAccess ?? null}, ${opts.invitedBy} + ) as id + `; + if (!row) throw new Error("create_space_invitation returned no row"); + return row.id as string; + }, + + async listSpaceInvitations(spaceId) { + const rows = await sql` + select * from ${sch}.list_space_invitations(${spaceId}) + `; + return rows.map( + (r): SpaceInvitation => ({ + id: r.id as string, + email: r.email as string, + admin: Boolean(r.admin), + shareAccess: (r.share_access as AccessLevel | null) ?? null, + invitedBy: (r.invited_by as string | null) ?? null, + invitedByName: (r.invited_by_name as string | null) ?? null, + createdAt: r.created_at as Date, + }), + ); + }, + + async revokeSpaceInvitation(spaceId, email) { + const [row] = await sql` + select ${sch}.revoke_space_invitation(${spaceId}, ${email}) as ok + `; + return Boolean(row?.ok); + }, + + async redeemSpaceInvitations(userId, email) { + const rows = await sql` + select * from ${sch}.redeem_space_invitations(${userId}, ${email}) + `; + return rows.map( + (r): RedeemedInvitation => ({ + spaceId: r.space_id as string, + slug: r.slug as string, + name: r.name as string, + admin: Boolean(r.admin), + shareAccess: (r.share_access as AccessLevel | null) ?? null, + }), + ); + }, + async withTransaction(fn: (db: CoreStore) => Promise): Promise { return sql.begin((tx) => fn(coreStore(tx as unknown as Sql, schema)), diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts index 2cc8f0c..3f930c1 100644 --- a/packages/engine/core/index.ts +++ b/packages/engine/core/index.ts @@ -16,7 +16,9 @@ export type { MemberSpace, Principal, PrincipalKind, + RedeemedInvitation, Space, + SpaceInvitation, SpacePrincipal, TreeAccess, TreeGrant, diff --git a/packages/engine/core/types.ts b/packages/engine/core/types.ts index 2dfc604..7628919 100644 --- a/packages/engine/core/types.ts +++ b/packages/engine/core/types.ts @@ -133,3 +133,31 @@ export interface ApiKeyInfo { createdAt: Date; expiresAt: Date | null; } + +/** + * A pending invitation to a space, keyed by invitee email (so an invite can be + * issued before the user registers). Redeemed at login against the verified + * email; see CoreStore.redeemSpaceInvitations. + */ +export interface SpaceInvitation { + id: string; + email: string; + /** Make the user a space admin on redemption. */ + admin: boolean; + /** Access granted at the shared root on redemption; null = no share grant. */ + shareAccess: AccessLevel | null; + /** The principal who issued the invite (null if it has since been deleted). */ + invitedBy: string | null; + /** Display name of the inviter (a user's name is their email), if resolvable. */ + invitedByName: string | null; + createdAt: Date; +} + +/** A space joined by redeeming an invitation. */ +export interface RedeemedInvitation { + spaceId: string; + slug: string; + name: string; + admin: boolean; + shareAccess: AccessLevel | null; +} From ba68c9184956150293af6be020134216c9a10db5 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 12:03:23 +0200 Subject: [PATCH 079/156] feat(server): redeem space invitations on verified login (INV-3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add redeemInvitationsForVerifiedLogin — a best-effort, idempotent helper that joins a user to every space they were invited to — and call it from the OAuth callback right after user resolution, next to the verified-email gate (invites are email-keyed, so redemption must require proven email ownership). A redemption failure is logged and swallowed so it never breaks the sign-in. Co-Authored-By: Claude Opus 4.8 --- .../server/handlers/auth.integration.test.ts | 108 ++++++++++++++++++ packages/server/handlers/auth.ts | 46 +++++++- 2 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 packages/server/handlers/auth.integration.test.ts diff --git a/packages/server/handlers/auth.integration.test.ts b/packages/server/handlers/auth.integration.test.ts new file mode 100644 index 0000000..73eb755 --- /dev/null +++ b/packages/server/handlers/auth.integration.test.ts @@ -0,0 +1,108 @@ +// Integration test for the OAuth callback's invitation-redemption hook (INV-3): +// redeemInvitationsForVerifiedLogin joins a user to every space they were invited +// to, and swallows failures so a redemption hiccup never breaks the sign-in. The +// redeem SQL/store itself is covered in the engine + migrate suites; here we cover +// the login-side glue against a real core schema. (Importing ./auth pulls in the +// OAuth handler but triggers no provider network calls — those only fire inside +// oauthCallbackHandler, which this test does not invoke.) +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/server/handlers/auth.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateCore } from "@memory.build/database"; +import { ACCESS, type CoreStore, coreStore } from "@memory.build/engine/core"; +import postgres, { type Sql } from "postgres"; +import { redeemInvitationsForVerifiedLogin } from "./auth"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; +const randomSlug = () => rand(12); + +let sql: Sql; +let coreSchema: string; +let core: CoreStore; + +async function v7(): Promise { + const [row] = await sql`select uuidv7() as id`; + return row?.id as string; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + await migrateCore(sql, { schema: coreSchema }); + core = coreStore(sql, coreSchema); +}); + +afterAll(async () => { + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +test("redeems pending invitations for a verified-login email", async () => { + const spaceId = await core.createSpace(randomSlug(), "Invited Space"); + const inviterId = await v7(); + await core.createUser(inviterId, `inviter_${rand(8)}@example.com`); + + const email = `invitee_${rand(8)}@example.com`; + await core.createSpaceInvitation(spaceId, email, { + admin: true, + shareAccess: ACCESS.write, + invitedBy: inviterId, + }); + + // the user already exists in core (the OAuth callback resolves/provisions the + // user before reaching the redemption hook) + const userId = await v7(); + await core.createUser(userId, email); + + const joined = await redeemInvitationsForVerifiedLogin(core, userId, email); + expect(joined).toBe(1); + + // joined the space as admin, with owner@home + write@share + const principals = await core.listSpacePrincipals(spaceId); + expect(principals.find((p) => p.id === userId)?.admin).toBe(true); + const ta = await core.buildTreeAccess(userId, spaceId); + expect(ta).toContainEqual({ + tree_path: `home.${userId.replace(/-/g, "")}`, + access: ACCESS.owner, + }); + expect(ta).toContainEqual({ tree_path: "share", access: ACCESS.write }); + + // invitation consumed; a second login is a no-op + expect(await core.listSpaceInvitations(spaceId)).toHaveLength(0); + expect(await redeemInvitationsForVerifiedLogin(core, userId, email)).toBe(0); +}); + +test("best-effort: a redemption failure is swallowed (does not throw)", async () => { + const spaceId = await core.createSpace(randomSlug(), "Space"); + const inviterId = await v7(); + await core.createUser(inviterId, `inviter_${rand(8)}@example.com`); + const email = `ghost_${rand(8)}@example.com`; + await core.createSpaceInvitation(spaceId, email, { + admin: false, + shareAccess: ACCESS.read, + invitedBy: inviterId, + }); + + // a user id that is not a core principal → add_principal_to_space FK fails + // inside redeem; the helper must swallow the error and report zero joins. + const orphanUserId = await v7(); + const joined = await redeemInvitationsForVerifiedLogin( + core, + orphanUserId, + email, + ); + expect(joined).toBe(0); + + // the failed redemption rolled back atomically: the invite is still pending + expect(await core.listSpaceInvitations(spaceId)).toHaveLength(1); +}); diff --git a/packages/server/handlers/auth.ts b/packages/server/handlers/auth.ts index 7f900a7..c047cd5 100644 --- a/packages/server/handlers/auth.ts +++ b/packages/server/handlers/auth.ts @@ -11,6 +11,7 @@ */ import type { AuthStore, OAuthProvider } from "@memory.build/auth"; +import { type CoreStore, coreStore } from "@memory.build/engine/core"; import { info, reportError } from "@pydantic/logfire-node"; import type { Sql } from "postgres"; import { buildAuthUrl, exchangeCode, fetchUserInfo } from "../auth/providers"; @@ -178,12 +179,43 @@ export async function deviceVerifyPostHandler( return new Response(null, { status: 302, headers: { Location: authUrl } }); } +/** + * Redeem pending space invitations for a just-verified login email: join the + * user to each invited space (owner@home + the per-invite share level). + * Idempotent, and best-effort — a redemption failure is logged and swallowed so + * it never fails the sign-in (the next login retries). Returns the number of + * spaces joined. The caller MUST have verified the user owns this email first + * (invitations are email-keyed; redeeming for an unverified email would let a + * caller claim invites sent to an address they don't control). + */ +export async function redeemInvitationsForVerifiedLogin( + core: CoreStore, + userId: string, + email: string, +): Promise { + try { + const joined = await core.redeemSpaceInvitations(userId, email); + if (joined.length > 0) { + info("Redeemed space invitations", { email, spaces: joined.length }); + } + return joined.length; + } catch (err) { + reportError( + "Invitation redemption failed (continuing sign-in)", + err as Error, + { email }, + ); + return 0; + } +} + /** * GET /api/v1/auth/callback/:provider — OAuth callback. * - * Resolves the user (account → verified email → provision), binds them to the - * device (status stays 'pending'), and shows the consent page. Authorization - * only happens when the human approves (POST /device/approve). + * Resolves the user (account → verified email → provision), redeems pending + * space invitations for the verified email, binds them to the device (status + * stays 'pending'), and shows the consent page. Authorization only happens when + * the human approves (POST /device/approve). */ export async function oauthCallbackHandler( request: Request, @@ -272,6 +304,14 @@ export async function oauthCallbackHandler( } } + // The email is verified (gated above) → proven owned, so redeem any pending + // space invitations sent to it (the user joins each invited space). + await redeemInvitationsForVerifiedLogin( + coreStore(ctx.db, ctx.coreSchema), + userId, + userInfo.email, + ); + // Bind the user; the device stays 'pending' until the human consents. await ctx.auth.bindDeviceUser(device.deviceCode, userId); return html( From ac317ae2b579bab235a0d193566e38a4561b4bb3 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 12:21:58 +0200 Subject: [PATCH 080/156] feat(server): space invitations + require admin for roster mutations (INV-4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add invite.create/list/revoke on the memory endpoint: invite.create adds an already-registered user immediately (instant access on their live session) and records a pending invitation for a not-yet-registered email (redeemed at first login); list shows pending invites; revoke deletes one. All invite.* and principal.add/remove now require space-admin (structural roster authority — owner@root is not enough), matching group management; the self-service own-agent add stays. Reads (principal.list/resolveByEmail) and grant.* remain manager/owner-level, with a TODO to review those. Export SHARE_NAMESPACE for the immediate-add share grant. Co-Authored-By: Claude Opus 4.8 --- TODO.md | 17 +++ packages/database/space/index.ts | 1 + packages/protocol/space/index.ts | 14 +++ packages/protocol/space/invitation.ts | 61 +++++++++ packages/server/rpc/memory/index.ts | 2 + packages/server/rpc/memory/invitation.ts | 104 +++++++++++++++ .../rpc/memory/management.integration.test.ts | 119 +++++++++++++++++- packages/server/rpc/memory/principal.ts | 11 +- packages/server/rpc/memory/support.ts | 16 +++ 9 files changed, 338 insertions(+), 7 deletions(-) create mode 100644 packages/protocol/space/invitation.ts create mode 100644 packages/server/rpc/memory/invitation.ts diff --git a/TODO.md b/TODO.md index 9e8a8bd..8a7f9df 100644 --- a/TODO.md +++ b/TODO.md @@ -224,3 +224,20 @@ but unproven at runtime. typecheck errors, and add an end-to-end check that the `me serve` `/rpc` proxy reaches the memory endpoint. Decide whether `packages/web` should be in CI / the root typecheck. + +## Review remaining `requireSpaceManager` endpoints (post INV-4) + +INV-4 moved the structural roster mutations (`principal.add`, `principal.remove`) +and all `invite.*` to `requireSpaceAdmin` (admin only — owner@root is not enough), +matching group management. The remaining manager-gated endpoints +(`requireSpaceManager` / `isSpaceManager` = admin **or** owner@root) were left +as-is and should be reviewed for the same admin-vs-manager question: + +- [ ] `principal.list` and `principal.resolveByEmail` + (`rpc/memory/principal.ts`) — reads of the roster. Decide whether viewing + the roster / resolving a user by email should be admin-only, or whether + owner@root is fine (current behavior). +- [ ] `grant.list` (`rpc/memory/grant.ts`), plus the `isSpaceManager` branch in + `requireGrantAuthority` used by `grant.set` / `grant.remove`. These are + data-access (owner@path) operations, so manager/owner is probably correct — + confirm they should *not* become admin-only. diff --git a/packages/database/space/index.ts b/packages/database/space/index.ts index 5372852..ff19100 100644 --- a/packages/database/space/index.ts +++ b/packages/database/space/index.ts @@ -10,6 +10,7 @@ export { homePrefix, normalizeTreeFilter, normalizeTreePath, + SHARE_NAMESPACE, TreePathError, type TreePathOptions, } from "./path"; diff --git a/packages/protocol/space/index.ts b/packages/protocol/space/index.ts index abb9c3a..39c723e 100644 --- a/packages/protocol/space/index.ts +++ b/packages/protocol/space/index.ts @@ -44,6 +44,14 @@ import { groupRenameParams, groupRenameResult, } from "./group.ts"; +import { + inviteCreateParams, + inviteCreateResult, + inviteListParams, + inviteListResult, + inviteRevokeParams, + inviteRevokeResult, +} from "./invitation.ts"; import { principalAddParams, principalAddResult, @@ -58,6 +66,7 @@ import { export * from "./api-key.ts"; export * from "./grant.ts"; export * from "./group.ts"; +export * from "./invitation.ts"; export * from "./principal.ts"; function method( @@ -102,6 +111,11 @@ export const spaceMethods = { "grant.remove": method(grantRemoveParams, grantRemoveResult), "grant.list": method(grantListParams, grantListResult), + // Invitations (3) — email-keyed; adds existing users now, else pending + "invite.create": method(inviteCreateParams, inviteCreateResult), + "invite.list": method(inviteListParams, inviteListResult), + "invite.revoke": method(inviteRevokeParams, inviteRevokeResult), + // Api keys (4) "apiKey.create": method(apiKeyCreateParams, apiKeyCreateResult), "apiKey.list": method(apiKeyListParams, apiKeyListResult), diff --git a/packages/protocol/space/invitation.ts b/packages/protocol/space/invitation.ts new file mode 100644 index 0000000..d3f8fa3 --- /dev/null +++ b/packages/protocol/space/invitation.ts @@ -0,0 +1,61 @@ +/** + * Space invitation method schemas (invite.*). + * + * Invitations are keyed by invitee email so an invite can be issued before the + * user registers. Inviting an *already-registered* user adds them to the space + * immediately (instant access on their existing session); inviting a not-yet- + * registered email records a pending invitation, redeemed at their first + * verified login. Each invite carries whether to make the user a space admin and + * an optional share level (read/write/owner at the shared root; null = none). + */ +import { z } from "zod"; +import { emailSchema } from "../fields.ts"; +import { accessLevelSchema } from "./grant.ts"; + +/** A pending invitation to the space. */ +export const spaceInvitationResponse = z.object({ + id: z.string(), + email: z.string(), + admin: z.boolean(), + /** Share-root access granted on redemption; null = no share grant. */ + shareAccess: accessLevelSchema.nullable(), + invitedBy: z.string().nullable(), + invitedByName: z.string().nullable(), + createdAt: z.string(), +}); +export type SpaceInvitationResponse = z.infer; + +// invite.create — invite by email; adds an existing user now, else records a +// pending invite. `admin` defaults false; `shareAccess` null/omitted = no share. +export const inviteCreateParams = z.object({ + email: emailSchema, + admin: z.boolean().optional(), + shareAccess: accessLevelSchema.nullable().optional(), +}); +export type InviteCreateParams = z.infer; + +export const inviteCreateResult = z.object({ + /** True when the invitee was an existing user and was added to the space now. */ + applied: z.boolean(), + /** The pending invitation id (null when applied immediately). */ + invitationId: z.string().nullable(), + /** The principal added now (null when deferred to a pending invitation). */ + principalId: z.string().nullable(), +}); +export type InviteCreateResult = z.infer; + +// invite.list — pending invitations for the space +export const inviteListParams = z.object({}); +export type InviteListParams = z.infer; + +export const inviteListResult = z.object({ + invitations: z.array(spaceInvitationResponse), +}); +export type InviteListResult = z.infer; + +// invite.revoke — delete a pending invitation by email +export const inviteRevokeParams = z.object({ email: emailSchema }); +export type InviteRevokeParams = z.infer; + +export const inviteRevokeResult = z.object({ revoked: z.boolean() }); +export type InviteRevokeResult = z.infer; diff --git a/packages/server/rpc/memory/index.ts b/packages/server/rpc/memory/index.ts index 83987fe..95df6f4 100644 --- a/packages/server/rpc/memory/index.ts +++ b/packages/server/rpc/memory/index.ts @@ -9,6 +9,7 @@ import type { MethodRegistry } from "../types"; import { apiKeyMethods } from "./api-key"; import { grantMethods } from "./grant"; import { groupMethods } from "./group"; +import { invitationMethods } from "./invitation"; import { memoryDataMethods } from "./memory"; import { principalMethods } from "./principal"; @@ -27,5 +28,6 @@ export const memoryMethods: MethodRegistry = new Map([ ...principalMethods, ...groupMethods, ...grantMethods, + ...invitationMethods, ...apiKeyMethods, ]); diff --git a/packages/server/rpc/memory/invitation.ts b/packages/server/rpc/memory/invitation.ts new file mode 100644 index 0000000..b117a01 --- /dev/null +++ b/packages/server/rpc/memory/invitation.ts @@ -0,0 +1,104 @@ +/** + * Space invitation handlers (invite.*). + * + * Inviting an *already-registered* user adds them to the space immediately — + * access is recomputed per request (build_tree_access), so it takes effect on + * their existing session without a re-login. Inviting a not-yet-registered email + * records a pending invitation, redeemed at their first verified login (see + * redeemInvitationsForVerifiedLogin). Both paths grant owner@home and, when a + * share level is set, that level at the shared root. + * + * Authority: all three methods require space-admin (structural authority over + * the roster, like group management — owner@root alone is not enough). Inviting + * people, optionally as admins, is a deliberate structural act. + */ +import { SHARE_NAMESPACE } from "@memory.build/database"; +import type { + InviteCreateParams, + InviteCreateResult, + InviteListParams, + InviteListResult, + InviteRevokeParams, + InviteRevokeResult, +} from "@memory.build/protocol/space"; +import { + inviteCreateParams, + inviteListParams, + inviteRevokeParams, +} from "@memory.build/protocol/space"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + guardCore, + requireSpaceAdmin, + toSpaceInvitationResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +async function inviteCreate( + params: InviteCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const admin = params.admin ?? false; + const shareAccess = params.shareAccess ?? null; + + // Already-registered user → add them now (instant access on their existing + // session). Not-yet-registered → a pending invite, redeemed at first login. + const existing = await ctx.core.getUserByName(params.email); + if (existing) { + await guardCore(async () => { + await ctx.core.addPrincipalToSpace(ctx.space.id, existing.id, admin); + if (shareAccess !== null) { + await ctx.core.grantTreeAccess( + ctx.space.id, + existing.id, + SHARE_NAMESPACE, + shareAccess, + ); + } + }); + return { applied: true, invitationId: null, principalId: existing.id }; + } + + const invitationId = await guardCore(() => + ctx.core.createSpaceInvitation(ctx.space.id, params.email, { + admin, + shareAccess, + invitedBy: ctx.principalId, + }), + ); + return { applied: false, invitationId, principalId: null }; +} + +async function inviteList( + _params: InviteListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const invitations = await ctx.core.listSpaceInvitations(ctx.space.id); + return { invitations: invitations.map(toSpaceInvitationResponse) }; +} + +async function inviteRevoke( + params: InviteRevokeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const revoked = await guardCore(() => + ctx.core.revokeSpaceInvitation(ctx.space.id, params.email), + ); + return { revoked }; +} + +export const invitationMethods = buildRegistry() + .register("invite.create", inviteCreateParams, inviteCreate) + .register("invite.list", inviteListParams, inviteList) + .register("invite.revoke", inviteRevokeParams, inviteRevoke) + .build(); diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index b382ed9..d24acb6 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -93,6 +93,14 @@ function makeAgent(owner: string): Promise { .createAgent(owner, `agent_${rand(6)}`); } +/** Create a registered user with a known email (the invite key), returning its id. */ +async function makeUserWithEmail(email: string): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await engineCore.coreStore(sql, coreSchema).createUser(id, email); + return id; +} + beforeAll(async () => { sql = postgres(URL, { onnotice: () => {} }); authSchema = `auth_test_${rand(8)}`; @@ -227,6 +235,101 @@ test("grant: set / list / remove", async () => { ).toBe(true); }); +test("invite.create: a not-yet-registered email creates a pending invite; list + revoke", async () => { + const email = `newcomer_${rand(8)}@example.com`; + const res = await call<{ + applied: boolean; + invitationId: string | null; + principalId: string | null; + }>("invite.create", { email, admin: false, shareAccess: 1 }); + expect(res.applied).toBe(false); + expect(res.invitationId).toBeTruthy(); + expect(res.principalId).toBeNull(); + + const { invitations } = await call<{ + invitations: { + email: string; + shareAccess: number | null; + invitedByName: string | null; + }[]; + }>("invite.list", {}); + expect(invitations).toHaveLength(1); + expect(invitations[0]?.email).toBe(email); + expect(invitations[0]?.shareAccess).toBe(1); + expect(invitations[0]?.invitedByName).toBe(ownerEmail); // the owner invited + + expect( + (await call<{ revoked: boolean }>("invite.revoke", { email })).revoked, + ).toBe(true); + expect( + (await call<{ invitations: unknown[] }>("invite.list", {})).invitations, + ).toHaveLength(0); +}); + +test("invite.create: an already-registered user is added immediately (no pending invite)", async () => { + const email = `existing_${rand(8)}@example.com`; + const existingId = await makeUserWithEmail(email); + + const res = await call<{ + applied: boolean; + invitationId: string | null; + principalId: string | null; + }>("invite.create", { email, admin: true, shareAccess: 2 }); + expect(res.applied).toBe(true); + expect(res.principalId).toBe(existingId); + expect(res.invitationId).toBeNull(); + + // they are a space admin now, with owner@home + write@share + const core = engineCore.coreStore(sql, coreSchema); + const principals = await core.listSpacePrincipals(space.id); + expect(principals.find((p) => p.id === existingId)?.admin).toBe(true); + const ta = await core.buildTreeAccess(existingId, space.id); + expect(ta).toContainEqual({ + tree_path: `home.${existingId.replace(/-/g, "")}`, + access: 3, + }); + expect(ta).toContainEqual({ tree_path: "share", access: 2 }); + + // joined → not shown as a pending invitation + expect( + (await call<{ invitations: unknown[] }>("invite.list", {})).invitations, + ).toHaveLength(0); +}); + +test("invite.* require space-admin authority (owner@root is not enough)", async () => { + // a plain member with no authority + const plain = await makeUser(); + const asPlain = { + principalId: plain, + treeAccess: [] as TreeAccess, + admin: false, + }; + await expectAppError(call("invite.list", {}, asPlain), "FORBIDDEN"); + + // a member who owns the whole data tree (owner@root) but is NOT a space admin + // is still forbidden — inviting is structural, like group management + const rootOwner = await makeUser(); + await call("principal.add", { principalId: rootOwner }); + await call("grant.set", { principalId: rootOwner, treePath: "", access: 3 }); + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(rootOwner, space.id); + const asOwner = { principalId: rootOwner, treeAccess: ta, admin: false }; + await expectAppError( + call( + "invite.create", + { email: `x_${rand(8)}@example.com`, admin: false, shareAccess: 1 }, + asOwner, + ), + "FORBIDDEN", + ); + await expectAppError(call("invite.list", {}, asOwner), "FORBIDDEN"); + await expectAppError( + call("invite.revoke", { email: `x_${rand(8)}@example.com` }, asOwner), + "FORBIDDEN", + ); +}); + test("apiKey: create (agent-only) / list / get / delete", async () => { // agent lifecycle is the user endpoint's job; here the owner brings an agent // into the space, then mints its (space-bound) key. @@ -424,7 +527,7 @@ test("group.listForMember: an agent's owner can list its groups", async () => { ); }); -test("group management requires admin — owner@root is not enough", async () => { +test("structural mutations require admin — owner@root is not enough", async () => { // a member who owns the whole data tree (owner@root) but is NOT a space admin const member = await makeUser(); await call("principal.add", { principalId: member }); @@ -434,12 +537,22 @@ test("group management requires admin — owner@root is not enough", async () => .buildTreeAccess(member, space.id); const as = { principalId: member, treeAccess: ta, admin: false }; - // owner@root can manage the roster and grant access (it's their data) + // owner@root can READ the roster and manage grants (it's their data)... expect( (await call<{ principals: unknown[] }>("principal.list", {}, as)).principals .length, ).toBeGreaterThan(0); - // but groups are structural — admin only + // ...but structural changes are admin-only: adding/removing roster members and + // creating groups. Owning the data tree is not structural authority. + const stranger = await makeUser(); + await expectAppError( + call("principal.add", { principalId: stranger }, as), + "FORBIDDEN", + ); + await expectAppError( + call("principal.remove", { principalId: member }, as), + "FORBIDDEN", + ); await expectAppError(call("group.create", { name: "g" }, as), "FORBIDDEN"); }); diff --git a/packages/server/rpc/memory/principal.ts b/packages/server/rpc/memory/principal.ts index 286c925..54fcd35 100644 --- a/packages/server/rpc/memory/principal.ts +++ b/packages/server/rpc/memory/principal.ts @@ -22,6 +22,7 @@ import type { HandlerContext } from "../types"; import { callerOwnsAgentGlobal, guardCore, + requireSpaceAdmin, requireSpaceManager, toPrincipalResponse, toSpacePrincipalResponse, @@ -49,13 +50,14 @@ async function principalAdd( assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; // Bringing your OWN agent into a space is self-service (it stays capped by - // your access); adding anyone else requires space-owner authority. A member - // can't grant themselves admin on their own agent membership. + // your access); adding anyone else is a structural roster change that requires + // space-admin (owner@root is not enough). A member can't grant themselves admin + // on their own agent membership. const ownAgent = params.admin !== true && (await callerOwnsAgentGlobal(ctx, params.principalId)); if (!ownAgent) { - requireSpaceManager(ctx); + requireSpaceAdmin(ctx); } await guardCore(() => ctx.core.addPrincipalToSpace( @@ -73,7 +75,8 @@ async function principalRemove( ): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; - requireSpaceManager(ctx); + // Removing a roster member is structural, like adding — space-admin only. + requireSpaceAdmin(ctx); const removed = await guardCore(() => ctx.core.removePrincipalFromSpace(ctx.space.id, params.principalId), ); diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts index a2edbd4..4b6b097 100644 --- a/packages/server/rpc/memory/support.ts +++ b/packages/server/rpc/memory/support.ts @@ -16,6 +16,7 @@ import type { GroupMember, GroupMembership, Principal, + SpaceInvitation, SpacePrincipal, TreeGrant, } from "@memory.build/engine/core"; @@ -26,6 +27,7 @@ import type { GroupMembershipResponse, GroupResponse, PrincipalResponse, + SpaceInvitationResponse, SpacePrincipalResponse, TreeGrantResponse, } from "@memory.build/protocol/space"; @@ -307,3 +309,17 @@ export function toApiKeyInfoResponse(k: ApiKeyInfo): ApiKeyInfoResponse { expiresAt: k.expiresAt?.toISOString() ?? null, }; } + +export function toSpaceInvitationResponse( + i: SpaceInvitation, +): SpaceInvitationResponse { + return { + id: i.id, + email: i.email, + admin: i.admin, + shareAccess: i.shareAccess, + invitedBy: i.invitedBy, + invitedByName: i.invitedByName, + createdAt: i.createdAt.toISOString(), + }; +} From d4938dfa1af22f713b8027090e8c3d6ae3d5dc42 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 12:36:03 +0200 Subject: [PATCH 081/156] feat(cli): me space invite/list/revoke + client invite namespace (INV-5) Add the invite namespace (create/list/revoke) to the memory client, and the `me space invite` CLI: `invite [--admin] [--share none|read|write|owner]` (default read) which adds an already-registered user immediately or records a pending invite, plus `invite list` and `invite revoke `. Centralize the access-level name<->number translation as accessLevelName / parseAccessLevel in protocol/space/grant.ts (next to AccessLevel); me access and me space invite both use it instead of private copies. Co-Authored-By: Claude Opus 4.8 --- packages/cli/commands/access.ts | 29 ++---- packages/cli/commands/space.ts | 148 ++++++++++++++++++++++++++----- packages/client/index.ts | 1 + packages/client/memory.ts | 18 ++++ packages/protocol/space/grant.ts | 28 ++++++ 5 files changed, 182 insertions(+), 42 deletions(-) diff --git a/packages/cli/commands/access.ts b/packages/cli/commands/access.ts index 8fc2ff1..b5a264e 100644 --- a/packages/cli/commands/access.ts +++ b/packages/cli/commands/access.ts @@ -12,6 +12,10 @@ * is a UUID, or a name (user = email, agent/group = display name). */ import * as clack from "@clack/prompts"; +import { + accessLevelName, + parseAccessLevel, +} from "@memory.build/protocol/space"; import { Command } from "commander"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; @@ -23,25 +27,6 @@ import { resolveSpacePrincipalId, } from "../util.ts"; -/** Access level: r = read (1), w = write (2), o = owner (3). */ -const LEVELS: Record = { - r: 1, - read: 1, - w: 2, - write: 2, - o: 3, - owner: 3, -}; -const LEVEL_NAME: Record = { - 1: "read", - 2: "write", - 3: "owner", -}; - -function parseLevel(input: string): 1 | 2 | 3 | null { - return LEVELS[input.toLowerCase()] ?? null; -} - function createAccessGrantCommand(): Command { return new Command("grant") .description("grant or update a principal's access at a tree path") @@ -59,7 +44,7 @@ function createAccessGrantCommand(): Command { requireSession(creds, fmt); requireSpace(creds, fmt); - const access = parseLevel(level); + const access = parseAccessLevel(level); if (!access) { handleError( new Error(`Invalid level '${level}'. Use r, w, or o.`), @@ -84,7 +69,7 @@ function createAccessGrantCommand(): Command { fmt, () => { clack.log.success( - `Granted ${LEVEL_NAME[access]} on '${path}' to ${principal}`, + `Granted ${accessLevelName(access)} on '${path}' to ${principal}`, ); }, ); @@ -173,7 +158,7 @@ function createAccessListCommand(): Command { grants.map((g) => [ names.get(g.principalId) ?? g.principalId, g.treePath === "" ? "(root)" : g.treePath, - LEVEL_NAME[g.access] ?? String(g.access), + accessLevelName(g.access), ]), ); }); diff --git a/packages/cli/commands/space.ts b/packages/cli/commands/space.ts index 969678b..d5752f4 100644 --- a/packages/cli/commands/space.ts +++ b/packages/cli/commands/space.ts @@ -6,12 +6,20 @@ * - me space create : create a space and make it active * - me space rename : rename a space's display label * - me space delete : delete a space and all its data - * - me space invite [--admin]: add an existing user to the active space + * - me space invite [--admin] [--share ]: invite by email (adds + * an existing user now, else a pending invite redeemed at their first login) + * - me space invite list: list pending invitations + * - me space invite revoke : revoke a pending invitation * * accepts a slug (exact) or a name (case-insensitive). The slug is the * immutable 12-char routing key; the name is the renamable display label. */ import * as clack from "@clack/prompts"; +import { + type AccessLevel, + accessLevelName, + parseAccessLevel, +} from "@memory.build/protocol/space"; import type { MemberSpaceResponse } from "@memory.build/protocol/user"; import { Command } from "commander"; import { createUserClient } from "../client.ts"; @@ -294,12 +302,33 @@ function createSpaceDeleteCommand(): Command { }); } -function createSpaceInviteCommand(): Command { - return new Command("invite") - .description("add an existing user to the active space (by email)") - .argument("", "the user's email") - .option("--admin", "grant space-admin (manage members and groups)") - .action(async (email: string, opts, cmd) => { +/** + * Map a `--share` value to the nullable access level: "none" → null (no share + * grant), otherwise read/write/owner via the shared parser. Exits on bad input. + */ +function parseShareLevel(value: string, fmt: OutputFormat): AccessLevel | null { + if (value.trim().toLowerCase() === "none") return null; + const level = parseAccessLevel(value); + if (level !== null) return level; + const msg = `Invalid --share value '${value}'. Use none, read, write, or owner.`; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); +} + +/** Display label for a stored share-access level (null → "none"). */ +function shareLabel(level: AccessLevel | null): string { + return level === null ? "none" : accessLevelName(level); +} + +function createSpaceInviteListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list pending invitations for the active space") + .action(async (_opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); @@ -307,32 +336,111 @@ function createSpaceInviteCommand(): Command { requireSpace(creds, fmt); const memory = buildMemoryClient(creds); + try { + const { invitations } = await memory.invite.list(); + output({ invitations }, fmt, () => { + if (invitations.length === 0) { + console.log(" No pending invitations."); + return; + } + table( + ["email", "admin", "share", "invited by", "created"], + invitations.map((i) => [ + i.email, + i.admin ? "yes" : "", + shareLabel(i.shareAccess), + i.invitedByName ?? "", + i.createdAt, + ]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} +function createSpaceInviteRevokeCommand(): Command { + return new Command("revoke") + .description("revoke a pending invitation by email") + .argument("", "the invitee's email") + .action(async (email: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); try { - const { principal } = await memory.principal.resolveByEmail({ email }); - if (!principal) { - const msg = `No user found with email '${email}'. They must sign in once before they can be added.`; - if (fmt === "text") { - clack.log.error(msg); + const result = await memory.invite.revoke({ email }); + output({ email, ...result }, fmt, () => { + if (result.revoked) { + clack.log.success(`Revoked the invitation for ${email}.`); } else { - output({ error: msg }, fmt, () => {}); + clack.log.warn(`No pending invitation for ${email}.`); } - process.exit(1); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceInviteCommand(): Command { + const invite = new Command("invite") + .description("invite a user to the active space by email") + .argument("[email]", "the invitee's email (omit when using a subcommand)") + .option("--admin", "make the user a space admin") + .option( + "--share ", + "shared-root access to grant: none | read | write | owner", + "read", + ) + .action(async (email: string | undefined, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + if (!email) { + const msg = + "An email is required: me space invite [--admin] [--share ]"; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg }, fmt, () => {}); } + process.exit(1); + } - const result = await memory.principal.add({ - principalId: principal.id, + const shareAccess = parseShareLevel(opts.share, fmt); + const memory = buildMemoryClient(creds); + try { + const result = await memory.invite.create({ + email, admin: opts.admin === true, + shareAccess, }); - output({ email, principalId: principal.id, ...result }, fmt, () => { - clack.log.success( - `Added ${email} to the space${opts.admin ? " as an admin" : ""}.`, - ); + output({ email, ...result }, fmt, () => { + if (result.applied) { + clack.log.success( + `Added ${email} to the space${opts.admin ? " as an admin" : ""}.`, + ); + } else { + clack.log.success( + `Invited ${email} — they'll join when they next sign in.`, + ); + } }); } catch (error) { handleError(error, fmt, { sessionServer: creds.server }); } }); + invite.addCommand(createSpaceInviteListCommand()); + invite.addCommand(createSpaceInviteRevokeCommand()); + return invite; } export function createSpaceCommand(): Command { diff --git a/packages/client/index.ts b/packages/client/index.ts index 14fff9a..b9e3e23 100644 --- a/packages/client/index.ts +++ b/packages/client/index.ts @@ -49,6 +49,7 @@ export { createMemoryClient, type GrantNamespace, type GroupNamespace, + type InviteNamespace, type MemoryClient, type MemoryClientOptions, type MemoryNamespace, diff --git a/packages/client/memory.ts b/packages/client/memory.ts index 08b447a..5bda57c 100644 --- a/packages/client/memory.ts +++ b/packages/client/memory.ts @@ -65,6 +65,12 @@ import type { GroupRemoveMemberResult, GroupRenameParams, GroupRenameResult, + InviteCreateParams, + InviteCreateResult, + InviteListParams, + InviteListResult, + InviteRevokeParams, + InviteRevokeResult, PrincipalAddParams, PrincipalAddResult, PrincipalListParams, @@ -138,6 +144,12 @@ export interface GrantNamespace { list(params?: GrantListParams): Promise; } +export interface InviteNamespace { + create(params: InviteCreateParams): Promise; + list(params?: InviteListParams): Promise; + revoke(params: InviteRevokeParams): Promise; +} + export interface ApiKeyNamespace { create(params: ApiKeyCreateParams): Promise; list(params: ApiKeyListParams): Promise; @@ -150,6 +162,7 @@ export interface MemoryClient { principal: PrincipalNamespace; group: GroupNamespace; grant: GrantNamespace; + invite: InviteNamespace; apiKey: ApiKeyNamespace; /** Update the bearer token (session or api key) at runtime. */ @@ -214,6 +227,11 @@ export function createMemoryClient( remove: (p) => rpc("grant.remove", p), list: (p) => rpc("grant.list", p ?? {}), }, + invite: { + create: (p) => rpc("invite.create", p), + list: (p) => rpc("invite.list", p ?? {}), + revoke: (p) => rpc("invite.revoke", p), + }, apiKey: { create: (p) => rpc("apiKey.create", p), list: (p) => rpc("apiKey.list", p), diff --git a/packages/protocol/space/grant.ts b/packages/protocol/space/grant.ts index bec967a..62bed10 100644 --- a/packages/protocol/space/grant.ts +++ b/packages/protocol/space/grant.ts @@ -16,6 +16,34 @@ export const accessLevelSchema = z.union([ ]); export type AccessLevel = z.infer; +/** The canonical name for an access level (1 → "read", 2 → "write", 3 → "owner"). */ +export function accessLevelName( + level: AccessLevel, +): "read" | "write" | "owner" { + return level === 1 ? "read" : level === 2 ? "write" : "owner"; +} + +/** + * Parse a human access-level string to its numeric level, or null if unknown. + * Accepts the full names (read/write/owner) and single-letter forms (r/w/o), + * case-insensitively. The "none" sentinel (no grant) is the caller's concern. + */ +export function parseAccessLevel(input: string): AccessLevel | null { + switch (input.trim().toLowerCase()) { + case "r": + case "read": + return 1; + case "w": + case "write": + return 2; + case "o": + case "owner": + return 3; + default: + return null; + } +} + export const treeGrantResponse = z.object({ principalId: z.string(), treePath: z.string(), From afd7deabbbf904dde5cfa850ec027cbfac236a1d Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 12:37:03 +0200 Subject: [PATCH 082/156] docs(todo): mark space invitations built; track deferred email delivery + expiry Co-Authored-By: Claude Opus 4.8 --- TODO.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/TODO.md b/TODO.md index 8a7f9df..3a64a57 100644 --- a/TODO.md +++ b/TODO.md @@ -28,16 +28,18 @@ that raw NOT_FOUND, so the user has to know to run `me agent add ` first. util.ts. (Auto-adding the agent was considered but skipped — silently changing space membership as a side effect of minting a key is surprising.) -## Space invitations - -The CLI spec includes `me space invite` / `invite list` / `invite revoke` -(invite a user by email into a space with an initial role/grant). Deferred from -4E — it's a new subsystem. - -- [ ] Design + build space-scoped invitations: a core table (space_id, email, - role/grant, token, status, expiry), RPC on the space endpoint - (invite.create/list/revoke + accept on the user endpoint), and the email/ - link delivery. Mirrors the device-flow consent UX where relevant. +## Space invitations: email delivery + expiry (deferred from v1) + +Space invitations are built (INV-1..5): an email-keyed `core.space_invitation`, +redeemed at verified login (or applied immediately for an already-registered +user), with `me space invite [--admin] [--share …]` / `invite list` / +`invite revoke`. Two pieces were intentionally deferred from v1: + +- [ ] Email/link delivery — v1 sends no notification; a pending invite is only + acted on when the invitee next signs in (auto-redeemed) or is told out of + band. Send an invitation email with a sign-in link. +- [ ] Invite expiry — `space_invitation` has no expiry; a pending invite lives + until redeemed or revoked. Add an expiry column (+ a sweep) if wanted. ## Worker: call space SQL functions instead of raw queries From 8dd409ba64aaba3c936e847b10109001d7e84dc9 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 13:05:59 +0200 Subject: [PATCH 083/156] refactor(server): member-accessible principal.resolve/lookup; principal.list admin-only Split roster access by intent: add principal.resolve (name->id) and principal.lookup (id->name, batched), both available to any space member (targeted lookups, not enumeration), and gate principal.list (full enumeration) to space-admin. Point the CLI's name resolution at principal.resolve (so a subtree owner can grant by name) and me access list's id->name display at principal.lookup. Remove the now-dead principal.resolveByEmail (superseded by invite.create) and its orphaned response type/serializer. Co-Authored-By: Claude Opus 4.8 --- TODO.md | 17 ------ packages/cli/commands/access.ts | 9 ++- packages/cli/util.test.ts | 15 +++-- packages/cli/util.ts | 21 ++++--- packages/client/memory.ts | 16 +++-- packages/protocol/space/index.ts | 12 ++-- packages/protocol/space/principal.ts | 47 ++++++++------ .../rpc/memory/management.integration.test.ts | 61 +++++++++++++++---- packages/server/rpc/memory/principal.ts | 60 ++++++++++++------ packages/server/rpc/memory/support.ts | 14 ----- 10 files changed, 161 insertions(+), 111 deletions(-) diff --git a/TODO.md b/TODO.md index 3a64a57..6716a34 100644 --- a/TODO.md +++ b/TODO.md @@ -226,20 +226,3 @@ but unproven at runtime. typecheck errors, and add an end-to-end check that the `me serve` `/rpc` proxy reaches the memory endpoint. Decide whether `packages/web` should be in CI / the root typecheck. - -## Review remaining `requireSpaceManager` endpoints (post INV-4) - -INV-4 moved the structural roster mutations (`principal.add`, `principal.remove`) -and all `invite.*` to `requireSpaceAdmin` (admin only — owner@root is not enough), -matching group management. The remaining manager-gated endpoints -(`requireSpaceManager` / `isSpaceManager` = admin **or** owner@root) were left -as-is and should be reviewed for the same admin-vs-manager question: - -- [ ] `principal.list` and `principal.resolveByEmail` - (`rpc/memory/principal.ts`) — reads of the roster. Decide whether viewing - the roster / resolving a user by email should be admin-only, or whether - owner@root is fine (current behavior). -- [ ] `grant.list` (`rpc/memory/grant.ts`), plus the `isSpaceManager` branch in - `requireGrantAuthority` used by `grant.set` / `grant.remove`. These are - data-access (owner@path) operations, so manager/owner is probably correct — - confirm they should *not* become admin-only. diff --git a/packages/cli/commands/access.ts b/packages/cli/commands/access.ts index b5a264e..001cbe0 100644 --- a/packages/cli/commands/access.ts +++ b/packages/cli/commands/access.ts @@ -139,13 +139,12 @@ function createAccessListCommand(): Command { treePath: opts.path ?? null, }); - // Map principal ids → names for display (best-effort). + // Map principal ids → names for display (member-accessible lookup). const names = new Map(); - try { - const { principals } = await memory.principal.list({}); + const ids = [...new Set(grants.map((g) => g.principalId))]; + if (ids.length > 0) { + const { principals } = await memory.principal.lookup({ ids }); for (const p of principals) names.set(p.id, p.name); - } catch { - // listing principals may require more authority than listing grants } output({ grants }, fmt, () => { diff --git a/packages/cli/util.test.ts b/packages/cli/util.test.ts index 89c3eb0..1059066 100644 --- a/packages/cli/util.test.ts +++ b/packages/cli/util.test.ts @@ -17,18 +17,18 @@ const UUID = "019d694f-79f6-7595-8faf-b70b01c11f98"; // ============================================================================= describe("resolveSpacePrincipalId", () => { - test("returns a UUIDv7 as-is without listing principals", async () => { + test("returns a UUIDv7 as-is without resolving", async () => { const memory = { - principal: { list: mock(() => Promise.reject(new Error("unused"))) }, + principal: { resolve: mock(() => Promise.reject(new Error("unused"))) }, } as unknown as MemoryClient; expect(await resolveSpacePrincipalId(memory, UUID, "text")).toBe(UUID); - expect(memory.principal.list).not.toHaveBeenCalled(); + expect(memory.principal.resolve).not.toHaveBeenCalled(); }); - test("resolves a name via principal.list (with optional kind)", async () => { + test("resolves a name via principal.resolve (with optional kind)", async () => { const memory = { principal: { - list: mock(() => + resolve: mock(() => Promise.resolve({ principals: [{ id: UUID, kind: "g", name: "eng" }], }), @@ -38,7 +38,10 @@ describe("resolveSpacePrincipalId", () => { const id = await resolveSpacePrincipalId(memory, "eng", "text", "g"); expect(id).toBe(UUID); - expect(memory.principal.list).toHaveBeenCalledWith({ kind: "g" }); + expect(memory.principal.resolve).toHaveBeenCalledWith({ + name: "eng", + kind: "g", + }); }); }); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 1f62149..14d4230 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -81,9 +81,9 @@ export function buildMemoryClient( /** * Resolve a principal in the active space to its id. Accepts a UUIDv7 (used * as-is) or a name — for users the name is their email; for agents/groups it is - * the display name. Optionally constrained to a kind ('u' | 'a' | 'g'). Listing - * principals requires space-manager authority; callers without it should pass a - * UUID. Exits with an actionable error on miss / ambiguity. + * the display name. Optionally constrained to a kind ('u' | 'a' | 'g'). Uses + * principal.resolve (a targeted lookup any space member may call). Exits with an + * actionable error on miss / ambiguity. */ export async function resolveSpacePrincipalId( memory: MemoryClient, @@ -93,13 +93,13 @@ export async function resolveSpacePrincipalId( ): Promise { if (UUIDV7_RE.test(input)) return input; - const { principals } = await memory.principal.list(kind ? { kind } : {}); - const lower = input.toLowerCase(); - const matches = principals.filter((p) => p.name.toLowerCase() === lower); + const { principals } = await memory.principal.resolve( + kind ? { name: input, kind } : { name: input }, + ); - if (matches.length === 1 && matches[0]) return matches[0].id; + if (principals.length === 1 && principals[0]) return principals[0].id; - if (matches.length === 0) { + if (principals.length === 0) { const msg = `No ${kind === "g" ? "group" : "principal"} named '${input}' in this space.`; if (fmt === "text") { clack.log.error(msg); @@ -112,9 +112,10 @@ export async function resolveSpacePrincipalId( const msg = `Multiple principals named '${input}'. Use the id instead:`; if (fmt === "text") { clack.log.error(msg); - for (const m of matches) console.log(` ${m.name} (${m.kind}) — ${m.id}`); + for (const m of principals) + console.log(` ${m.name} (${m.kind}) — ${m.id}`); } else { - output({ error: msg, matches }, fmt, () => {}); + output({ error: msg, matches: principals }, fmt, () => {}); } process.exit(1); } diff --git a/packages/client/memory.ts b/packages/client/memory.ts index 5bda57c..76a0725 100644 --- a/packages/client/memory.ts +++ b/packages/client/memory.ts @@ -75,10 +75,12 @@ import type { PrincipalAddResult, PrincipalListParams, PrincipalListResult, + PrincipalLookupParams, + PrincipalLookupResult, PrincipalRemoveParams, PrincipalRemoveResult, - PrincipalResolveByEmailParams, - PrincipalResolveByEmailResult, + PrincipalResolveParams, + PrincipalResolveResult, } from "@memory.build/protocol/space"; import { rpcCall, type TransportConfig } from "./transport.ts"; @@ -118,9 +120,10 @@ export interface PrincipalNamespace { list(params?: PrincipalListParams): Promise; add(params: PrincipalAddParams): Promise; remove(params: PrincipalRemoveParams): Promise; - resolveByEmail( - params: PrincipalResolveByEmailParams, - ): Promise; + /** Resolve principals in the space by name (member-accessible). */ + resolve(params: PrincipalResolveParams): Promise; + /** Reverse-lookup principal ids → name/kind (member-accessible). */ + lookup(params: PrincipalLookupParams): Promise; } export interface GroupNamespace { @@ -210,7 +213,8 @@ export function createMemoryClient( list: (p) => rpc("principal.list", p ?? {}), add: (p) => rpc("principal.add", p), remove: (p) => rpc("principal.remove", p), - resolveByEmail: (p) => rpc("principal.resolveByEmail", p), + resolve: (p) => rpc("principal.resolve", p), + lookup: (p) => rpc("principal.lookup", p), }, group: { create: (p) => rpc("group.create", p), diff --git a/packages/protocol/space/index.ts b/packages/protocol/space/index.ts index 39c723e..db52a1f 100644 --- a/packages/protocol/space/index.ts +++ b/packages/protocol/space/index.ts @@ -57,10 +57,12 @@ import { principalAddResult, principalListParams, principalListResult, + principalLookupParams, + principalLookupResult, principalRemoveParams, principalRemoveResult, - principalResolveByEmailParams, - principalResolveByEmailResult, + principalResolveParams, + principalResolveResult, } from "./principal.ts"; export * from "./api-key.ts"; @@ -85,10 +87,8 @@ export const spaceMethods = { "principal.list": method(principalListParams, principalListResult), "principal.add": method(principalAddParams, principalAddResult), "principal.remove": method(principalRemoveParams, principalRemoveResult), - "principal.resolveByEmail": method( - principalResolveByEmailParams, - principalResolveByEmailResult, - ), + "principal.resolve": method(principalResolveParams, principalResolveResult), + "principal.lookup": method(principalLookupParams, principalLookupResult), // Groups (8) "group.create": method(groupCreateParams, groupCreateResult), diff --git a/packages/protocol/space/principal.ts b/packages/protocol/space/principal.ts index d989c65..2318df7 100644 --- a/packages/protocol/space/principal.ts +++ b/packages/protocol/space/principal.ts @@ -11,7 +11,7 @@ * api-key holders). */ import { z } from "zod"; -import { emailSchema, nameSchema, uuidv7Schema } from "../fields.ts"; +import { nameSchema, uuidv7Schema } from "../fields.ts"; /** Principal kind: user / group / agent. */ export const principalKindSchema = z.enum(["u", "g", "a"]); @@ -34,17 +34,13 @@ export const spacePrincipalResponse = z.object({ }); export type SpacePrincipalResponse = z.infer; -/** A resolved principal (used by principal.resolveByEmail). */ -export const principalResponse = z.object({ +/** A principal reference: the minimal shape returned by resolve / lookup. */ +export const principalRef = z.object({ id: z.string(), kind: principalKindSchema, name: z.string(), - ownerId: z.string().nullable(), - spaceId: z.string().nullable(), - createdAt: z.string(), - updatedAt: z.string().nullable(), }); -export type PrincipalResponse = z.infer; +export type PrincipalRef = z.infer; // principal.list export const principalListParams = z.object({ @@ -74,18 +70,33 @@ export type PrincipalRemoveParams = z.infer; export const principalRemoveResult = z.object({ removed: z.boolean() }); export type PrincipalRemoveResult = z.infer; -// principal.resolveByEmail — find a global user by email (to add to the space) -export const principalResolveByEmailParams = z.object({ email: emailSchema }); -export type PrincipalResolveByEmailParams = z.infer< - typeof principalResolveByEmailParams ->; +// principal.resolve — resolve principals in this space by exact name +// (case-insensitive), optionally constrained to a kind. Available to any space +// member: a targeted name->id lookup, not roster enumeration (that is +// principal.list). Returns all matches so the caller can detect ambiguity. +export const principalResolveParams = z.object({ + name: z.string().min(1), + kind: principalKindSchema.optional().nullable(), +}); +export type PrincipalResolveParams = z.infer; + +export const principalResolveResult = z.object({ + principals: z.array(principalRef), +}); +export type PrincipalResolveResult = z.infer; + +// principal.lookup — reverse lookup: resolve a batch of principal ids to their +// names/kinds (for display, e.g. grant listings). Available to any space member; +// only ids that are in the space come back (you cannot enumerate by guessing). +export const principalLookupParams = z.object({ + ids: z.array(uuidv7Schema), +}); +export type PrincipalLookupParams = z.infer; -export const principalResolveByEmailResult = z.object({ - principal: principalResponse.nullable(), +export const principalLookupResult = z.object({ + principals: z.array(principalRef), }); -export type PrincipalResolveByEmailResult = z.infer< - typeof principalResolveByEmailResult ->; +export type PrincipalLookupResult = z.infer; // shared by agent.* / group.* mutation results export { nameSchema }; diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index d24acb6..2b64c7f 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -139,19 +139,13 @@ beforeEach(async () => { .buildTreeAccess(r.userId, r.spaceId); }); -test("principal: list / resolveByEmail / add / remove", async () => { +test("principal: list / add / remove", async () => { const listed = await call<{ principals: { id: string; admin: boolean }[] }>( "principal.list", {}, ); expect(listed.principals.some((m) => m.id === ownerId && m.admin)).toBe(true); - const resolved = await call<{ principal: { id: string } | null }>( - "principal.resolveByEmail", - { email: ownerEmail }, - ); - expect(resolved.principal?.id).toBe(ownerId); - const other = await makeUser(); expect( (await call<{ added: boolean }>("principal.add", { principalId: other })) @@ -171,6 +165,49 @@ test("principal: list / resolveByEmail / add / remove", async () => { ).toBe(true); }); +test("principal.resolve / lookup are available to non-admin members (list is admin-only)", async () => { + const email = `target_${rand(8)}@example.com`; + const targetId = await makeUserWithEmail(email); + await call("principal.add", { principalId: targetId }); // added by the admin owner + + // a non-admin caller: resolve/lookup have no authority gate beyond being in the + // space, so they work; principal.list (full enumeration) does not. + const asMember = { + principalId: targetId, + treeAccess: [{ tree_path: "x", access: 1 }] as TreeAccess, + admin: false, + }; + + const resolved = await call<{ principals: { id: string; name: string }[] }>( + "principal.resolve", + { name: email.toUpperCase() }, // case-insensitive + asMember, + ); + expect(resolved.principals).toHaveLength(1); + expect(resolved.principals[0]?.id).toBe(targetId); + + const looked = await call<{ principals: { id: string; name: string }[] }>( + "principal.lookup", + { ids: [targetId] }, + asMember, + ); + expect(looked.principals[0]?.name).toBe(email); + + // a name that isn't in the space resolves to nothing + expect( + ( + await call<{ principals: unknown[] }>( + "principal.resolve", + { name: `nobody_${rand(8)}@example.com` }, + asMember, + ) + ).principals, + ).toHaveLength(0); + + // full enumeration stays admin-only + await expectAppError(call("principal.list", {}, asMember), "FORBIDDEN"); +}); + test("group: create / list / members / rename / delete", async () => { const { id: groupId } = await call<{ id: string }>("group.create", { name: "eng", @@ -537,13 +574,13 @@ test("structural mutations require admin — owner@root is not enough", async () .buildTreeAccess(member, space.id); const as = { principalId: member, treeAccess: ta, admin: false }; - // owner@root can READ the roster and manage grants (it's their data)... + // owner@root can manage grants on its data — e.g. list them... expect( - (await call<{ principals: unknown[] }>("principal.list", {}, as)).principals - .length, + (await call<{ grants: unknown[] }>("grant.list", {}, as)).grants.length, ).toBeGreaterThan(0); - // ...but structural changes are admin-only: adding/removing roster members and - // creating groups. Owning the data tree is not structural authority. + // ...but the roster (enumeration + add/remove) and groups are admin-only: + // owning the data tree is not structural authority. + await expectAppError(call("principal.list", {}, as), "FORBIDDEN"); const stranger = await makeUser(); await expectAppError( call("principal.add", { principalId: stranger }, as), diff --git a/packages/server/rpc/memory/principal.ts b/packages/server/rpc/memory/principal.ts index 54fcd35..8369872 100644 --- a/packages/server/rpc/memory/principal.ts +++ b/packages/server/rpc/memory/principal.ts @@ -6,16 +6,19 @@ import type { PrincipalAddResult, PrincipalListParams, PrincipalListResult, + PrincipalLookupParams, + PrincipalLookupResult, PrincipalRemoveParams, PrincipalRemoveResult, - PrincipalResolveByEmailParams, - PrincipalResolveByEmailResult, + PrincipalResolveParams, + PrincipalResolveResult, } from "@memory.build/protocol/space"; import { principalAddParams, principalListParams, + principalLookupParams, principalRemoveParams, - principalResolveByEmailParams, + principalResolveParams, } from "@memory.build/protocol/space"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; @@ -23,8 +26,6 @@ import { callerOwnsAgentGlobal, guardCore, requireSpaceAdmin, - requireSpaceManager, - toPrincipalResponse, toSpacePrincipalResponse, } from "./support"; import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; @@ -35,7 +36,9 @@ async function principalList( ): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; - requireSpaceManager(ctx); + // Enumerating the whole roster is structural — admin only. (Targeted name / id + // lookups for any member are principal.resolve / principal.lookup.) + requireSpaceAdmin(ctx); const principals = await ctx.core.listSpacePrincipals( ctx.space.id, params.kind ?? undefined, @@ -83,24 +86,47 @@ async function principalRemove( return { removed }; } -async function principalResolveByEmail( - params: PrincipalResolveByEmailParams, +async function principalResolve( + params: PrincipalResolveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // No authority gate beyond space participation: reaching this handler means the + // caller has access in this space (the authenticate-space membership gate). This + // is a targeted name->id lookup, not roster enumeration (that is principal.list). + const principals = await ctx.core.listSpacePrincipals( + ctx.space.id, + params.kind ?? undefined, + ); + const lower = params.name.trim().toLowerCase(); + const matches = principals + .filter((p) => p.name.toLowerCase() === lower) + .map((p) => ({ id: p.id, kind: p.kind, name: p.name })); + return { principals: matches }; +} + +async function principalLookup( + params: PrincipalLookupParams, context: HandlerContext, -): Promise { +): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; - requireSpaceManager(ctx); - const principal = await ctx.core.getUserByName(params.email); - return { principal: principal ? toPrincipalResponse(principal) : null }; + // Member-accessible reverse lookup (id -> name/kind) for display; only ids that + // are in the space come back. Same gating rationale as principalResolve. + const ids = new Set(params.ids); + if (ids.size === 0) return { principals: [] }; + const principals = await ctx.core.listSpacePrincipals(ctx.space.id); + const found = principals + .filter((p) => ids.has(p.id)) + .map((p) => ({ id: p.id, kind: p.kind, name: p.name })); + return { principals: found }; } export const principalMethods = buildRegistry() .register("principal.list", principalListParams, principalList) .register("principal.add", principalAddParams, principalAdd) .register("principal.remove", principalRemoveParams, principalRemove) - .register( - "principal.resolveByEmail", - principalResolveByEmailParams, - principalResolveByEmail, - ) + .register("principal.resolve", principalResolveParams, principalResolve) + .register("principal.lookup", principalLookupParams, principalLookup) .build(); diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts index 4b6b097..0df1d9e 100644 --- a/packages/server/rpc/memory/support.ts +++ b/packages/server/rpc/memory/support.ts @@ -15,7 +15,6 @@ import type { Group, GroupMember, GroupMembership, - Principal, SpaceInvitation, SpacePrincipal, TreeGrant, @@ -26,7 +25,6 @@ import type { GroupMemberResponse, GroupMembershipResponse, GroupResponse, - PrincipalResponse, SpaceInvitationResponse, SpacePrincipalResponse, TreeGrantResponse, @@ -244,18 +242,6 @@ export function toSpacePrincipalResponse( }; } -export function toPrincipalResponse(p: Principal): PrincipalResponse { - return { - id: p.id, - kind: p.kind, - name: p.name, - ownerId: p.ownerId, - spaceId: p.spaceId, - createdAt: p.createdAt.toISOString(), - updatedAt: p.updatedAt?.toISOString() ?? null, - }; -} - export function toGroupResponse(g: Group): GroupResponse { return { id: g.id, From 9b590ed1d59ea39be390816464601485df12d76f Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 14:03:15 +0200 Subject: [PATCH 084/156] feat(server): space creator gets owner@home + owner@share + admin, not owner@root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A space creator is now a space admin and owner of its own home (via add_principal_to_space) and the shared root `share` — not owner@root — so by default it sees /share and ~, not other members' homes (and can self-grant owner@root as an admin). Unify the policy in addSpaceCreator(), shared by provisionUser and space.create so they can't drift. Fixtures: memory data-plane tests write under share. (the normalization test grants itself root); provision/auth assert the home+share shape. Co-Authored-By: Claude Opus 4.8 --- .../authenticate-space.integration.test.ts | 7 +- packages/server/provision.integration.test.ts | 11 +- packages/server/provision.ts | 40 +++++-- .../rpc/memory/memory.integration.test.ts | 104 +++++++++++------- .../server/rpc/user/agent.integration.test.ts | 5 +- packages/server/rpc/user/space.ts | 16 +-- 6 files changed, 118 insertions(+), 65 deletions(-) diff --git a/packages/server/middleware/authenticate-space.integration.test.ts b/packages/server/middleware/authenticate-space.integration.test.ts index 23a48f4..1d25faa 100644 --- a/packages/server/middleware/authenticate-space.integration.test.ts +++ b/packages/server/middleware/authenticate-space.integration.test.ts @@ -102,8 +102,9 @@ test("session: member with owner grant resolves space + treeAccess", async () => expect(result.context.space.id).toBe(p.spaceId); expect(result.context.principalId).toBe(p.userId); expect(result.context.apiKeyId).toBeNull(); + // the creator owns the shared root (and its own home), not owner@root expect(result.context.treeAccess).toContainEqual({ - tree_path: engineCore.ROOT_PATH, + tree_path: "share", access: engineCore.ACCESS.owner, }); } @@ -115,10 +116,12 @@ test("api key: agent of the space resolves with apiKeyId set", async () => { const agentId = await core.createAgent(p.userId, `agent-${rand()}`); await core.addPrincipalToSpace(p.spaceId, agentId); + // grant within the owner's access (it owns `share`) so the agent's clamped + // effective access is non-empty — the owner is no longer owner@root. await core.grantTreeAccess( p.spaceId, agentId, - engineCore.ROOT_PATH, + "share", engineCore.ACCESS.read, ); const key = await core.createApiKey(agentId, "ci"); diff --git a/packages/server/provision.integration.test.ts b/packages/server/provision.integration.test.ts index b9f94ba..8abfd59 100644 --- a/packages/server/provision.integration.test.ts +++ b/packages/server/provision.integration.test.ts @@ -95,11 +95,20 @@ test("provisions a new user: identity + principal + space + owner grant", async expect(space?.id).toBe(r.spaceId); expect(await schemaExists(`me_${r.spaceSlug}`)).toBe(true); - // owner of the space root + // the creator's default grants: owner of its home + the shared root (`share`), + // but NOT owner@root const ta = await engineCore .coreStore(sql, coreSchema) .buildTreeAccess(r.userId, r.spaceId); expect(ta).toContainEqual({ + tree_path: "share", + access: engineCore.ACCESS.owner, + }); + expect(ta).toContainEqual({ + tree_path: `home.${r.userId.replace(/-/g, "")}`, + access: engineCore.ACCESS.owner, + }); + expect(ta).not.toContainEqual({ tree_path: engineCore.ROOT_PATH, access: engineCore.ACCESS.owner, }); diff --git a/packages/server/provision.ts b/packages/server/provision.ts index 4f802c2..e3fd5c1 100644 --- a/packages/server/provision.ts +++ b/packages/server/provision.ts @@ -1,5 +1,9 @@ import { authStore, type OAuthProvider } from "@memory.build/auth"; -import { generateSlug, provisionSpace } from "@memory.build/database"; +import { + generateSlug, + provisionSpace, + SHARE_NAMESPACE, +} from "@memory.build/database"; import * as engineCore from "@memory.build/engine/core"; import type { Sql } from "postgres"; @@ -11,7 +15,8 @@ import type { Sql } from "postgres"; * - core.principal (kind 'u') sharing the SAME id as auth.users * - a default core.space + its me_ data schema (provisionSpace runs the * schema DDL inside this transaction) - * - the user's owner grant on the space root + * - the user as space admin + owner of its home and the shared root (`share`), + * not owner@root * * Because schema creation is transactional, any failure rolls the whole thing * back — no orphaned me_ schema, no cleanup code. No API key is minted: @@ -39,6 +44,28 @@ export interface ProvisionUserResult { spaceSlug: string; } +/** + * Grant a new space's creator its default access — shared by first-login + * provisioning and `space.create` so the two stay in lockstep. The creator + * becomes a space admin who owns its home (via add_principal_to_space) and the + * shared root (`share`), but NOT owner@root: it sees `/share` and its own `~`, + * not other members' homes. As an admin it can self-grant owner@root later if it + * wants the whole tree. Call inside the space-creation transaction. + */ +export async function addSpaceCreator( + core: engineCore.CoreStore, + spaceId: string, + userId: string, +): Promise { + await core.addPrincipalToSpace(spaceId, userId, true); // admin + owner@home + await core.grantTreeAccess( + spaceId, + userId, + SHARE_NAMESPACE, + engineCore.ACCESS.owner, + ); +} + export function provisionUser( sql: Sql, schemas: { auth: string; core: string }, @@ -63,14 +90,7 @@ export function provisionUser( const spaceId = await core.createSpace(slug, params.spaceName ?? "default"); await provisionSpace(tx, { slug }); // creates the me_ data schema - await core.addPrincipalToSpace(spaceId, userId, true); - // owner of the root path → the user owns the whole space - await core.grantTreeAccess( - spaceId, - userId, - engineCore.ROOT_PATH, - engineCore.ACCESS.owner, - ); + await addSpaceCreator(core, spaceId, userId); return { userId, spaceId, spaceSlug: slug }; }) as Promise; diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index 82d523d..3adb0b0 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -109,12 +109,26 @@ beforeEach(async () => { ); createdSpaceSchemas.push(`me_${r.spaceSlug}`); store = engineSpace.spaceStore(sql, `me_${r.spaceSlug}`); + // The default creator owns its home + the shared root (`share`), not the whole + // tree — so these tests write under `share.*`. (The normalization test, which + // needs arbitrary paths, elevates itself to owner@root.) treeAccess = await core.buildTreeAccess(r.userId, r.spaceId); space = { id: r.spaceId, slug: r.spaceSlug }; principalId = r.userId; }); test("~ home + lenient separators normalize on input, reverse-map on output", async () => { + // This test writes to arbitrary (non-home, non-share) paths to exercise path + // normalization, so elevate the owner to owner@root for it. + const core = engineCore.coreStore(sql, coreSchema); + await core.grantTreeAccess( + space.id, + principalId, + engineCore.ROOT_PATH, + engineCore.ACCESS.owner, + ); + treeAccess = await core.buildTreeAccess(principalId, space.id); + const home = `home.${principalId.replace(/-/g, "")}`; // `~/notes` (slash accepted on input) stores under the caller's home and @@ -159,7 +173,7 @@ test("~ home + lenient separators normalize on input, reverse-map on output", as test("create → get round-trips content/tree/meta and createdBy is null", async () => { const created = await call<{ id: string; createdBy: string | null }>( "memory.create", - { content: "hello world", tree: "notes.work", meta: { tag: "a" } }, + { content: "hello world", tree: "share.notes.work", meta: { tag: "a" } }, ); expect(created.createdBy).toBeNull(); @@ -171,7 +185,7 @@ test("create → get round-trips content/tree/meta and createdBy is null", async hasEmbedding: boolean; }>("memory.get", { id: created.id }); expect(got.content).toBe("hello world"); - expect(got.tree).toBe("notes.work"); + expect(got.tree).toBe("share.notes.work"); expect(got.meta).toEqual({ tag: "a" }); expect(got.hasEmbedding).toBe(false); }); @@ -179,6 +193,7 @@ test("create → get round-trips content/tree/meta and createdBy is null", async test("create with temporal round-trips as {start,end}", async () => { const created = await call<{ id: string }>("memory.create", { content: "temporal one", + tree: "share", temporal: { start: "2024-01-01T00:00:00Z", end: "2024-01-02T00:00:00Z" }, }); const got = await call<{ temporal: { start: string; end: string } | null }>( @@ -194,19 +209,20 @@ test("create with temporal round-trips as {start,end}", async () => { test("update patches fields", async () => { const created = await call<{ id: string }>("memory.create", { content: "before", - tree: "a", + tree: "share.a", }); const updated = await call<{ content: string; tree: string }>( "memory.update", - { id: created.id, content: "after", tree: "a.b" }, + { id: created.id, content: "after", tree: "share.a.b" }, ); expect(updated.content).toBe("after"); - expect(updated.tree).toBe("a.b"); + expect(updated.tree).toBe("share.a.b"); }); test("delete removes; get then NOT_FOUND", async () => { const created = await call<{ id: string }>("memory.create", { content: "doomed", + tree: "share", }); const res = await call<{ deleted: boolean }>("memory.delete", { id: created.id, @@ -224,14 +240,14 @@ test("get / delete unknown id → NOT_FOUND", async () => { test("batchCreate inserts all and is retrievable", async () => { const res = await call<{ ids: string[] }>("memory.batchCreate", { memories: [ - { content: "one", tree: "batch" }, - { content: "two", tree: "batch" }, - { content: "three", tree: "batch.sub" }, + { content: "one", tree: "share.batch" }, + { content: "two", tree: "share.batch" }, + { content: "three", tree: "share.batch.sub" }, ], }); expect(res.ids).toHaveLength(3); const count = await call<{ count: number }>("memory.countTree", { - tree: "batch", + tree: "share.batch", }); expect(count.count).toBe(3); }); @@ -239,93 +255,97 @@ test("batchCreate inserts all and is retrievable", async () => { test("tree returns descendant node counts under a path", async () => { await call("memory.batchCreate", { memories: [ - { content: "x", tree: "root.a" }, - { content: "y", tree: "root.a.deep" }, - { content: "z", tree: "root.b" }, + { content: "x", tree: "share.root.a" }, + { content: "y", tree: "share.root.a.deep" }, + { content: "z", tree: "share.root.b" }, ], }); const res = await call<{ nodes: { path: string; count: number }[] }>( "memory.tree", - { tree: "root" }, + { tree: "share.root" }, ); const byPath = Object.fromEntries(res.nodes.map((n) => [n.path, n.count])); - expect(byPath["root.a"]).toBe(2); - expect(byPath["root.a.deep"]).toBe(1); - expect(byPath["root.b"]).toBe(1); + expect(byPath["share.root.a"]).toBe(2); + expect(byPath["share.root.a.deep"]).toBe(1); + expect(byPath["share.root.b"]).toBe(1); // the base path itself is excluded - expect(byPath.root).toBeUndefined(); + expect(byPath["share.root"]).toBeUndefined(); }); test("tree respects levels depth limit", async () => { await call("memory.batchCreate", { - memories: [{ content: "deep", tree: "t.a.b.c" }], + memories: [{ content: "deep", tree: "share.t.a.b.c" }], }); const res = await call<{ nodes: { path: string }[] }>("memory.tree", { - tree: "t", + tree: "share.t", levels: 1, }); const paths = res.nodes.map((n) => n.path); - expect(paths).toContain("t.a"); - expect(paths).not.toContain("t.a.b"); + expect(paths).toContain("share.t.a"); + expect(paths).not.toContain("share.t.a.b"); }); test("move relocates a subtree (dryRun counts without moving)", async () => { await call("memory.batchCreate", { memories: [ - { content: "m1", tree: "src.x" }, - { content: "m2", tree: "src.y" }, + { content: "m1", tree: "share.src.x" }, + { content: "m2", tree: "share.src.y" }, ], }); const dry = await call<{ count: number }>("memory.move", { - source: "src", - destination: "dst", + source: "share.src", + destination: "share.dst", dryRun: true, }); expect(dry.count).toBe(2); - // still under src + // still under share.src expect( - (await call<{ count: number }>("memory.countTree", { tree: "src" })).count, + (await call<{ count: number }>("memory.countTree", { tree: "share.src" })) + .count, ).toBe(2); const moved = await call<{ count: number }>("memory.move", { - source: "src", - destination: "dst", + source: "share.src", + destination: "share.dst", }); expect(moved.count).toBe(2); expect( - (await call<{ count: number }>("memory.countTree", { tree: "src" })).count, + (await call<{ count: number }>("memory.countTree", { tree: "share.src" })) + .count, ).toBe(0); expect( - (await call<{ count: number }>("memory.countTree", { tree: "dst" })).count, + (await call<{ count: number }>("memory.countTree", { tree: "share.dst" })) + .count, ).toBe(2); }); test("deleteTree removes a subtree (dryRun counts without deleting)", async () => { await call("memory.batchCreate", { memories: [ - { content: "d1", tree: "gone.a" }, - { content: "d2", tree: "gone.b" }, + { content: "d1", tree: "share.gone.a" }, + { content: "d2", tree: "share.gone.b" }, ], }); const dry = await call<{ count: number }>("memory.deleteTree", { - tree: "gone", + tree: "share.gone", dryRun: true, }); expect(dry.count).toBe(2); const del = await call<{ count: number }>("memory.deleteTree", { - tree: "gone", + tree: "share.gone", }); expect(del.count).toBe(2); expect( - (await call<{ count: number }>("memory.countTree", { tree: "gone" })).count, + (await call<{ count: number }>("memory.countTree", { tree: "share.gone" })) + .count, ).toBe(0); }); test("search: fulltext (bm25) finds matching content", async () => { await call("memory.batchCreate", { memories: [ - { content: "the quick brown fox", tree: "s" }, - { content: "lazy dogs sleep", tree: "s" }, + { content: "the quick brown fox", tree: "share.s" }, + { content: "lazy dogs sleep", tree: "share.s" }, ], }); const res = await call<{ results: { content: string }[]; total: number }>( @@ -339,15 +359,15 @@ test("search: fulltext (bm25) finds matching content", async () => { test("search: tree filter only (no ranking) returns matches", async () => { await call("memory.batchCreate", { memories: [ - { content: "in scope", tree: "scope.a" }, - { content: "out of scope", tree: "other" }, + { content: "in scope", tree: "share.scope.a" }, + { content: "out of scope", tree: "share.other" }, ], }); const res = await call<{ results: { tree: string }[] }>("memory.search", { - tree: "scope", + tree: "share.scope", }); expect(res.results.length).toBe(1); - expect(res.results[0]?.tree).toBe("scope.a"); + expect(res.results[0]?.tree).toBe("share.scope.a"); }); test("search: grep alone is rejected", async () => { diff --git a/packages/server/rpc/user/agent.integration.test.ts b/packages/server/rpc/user/agent.integration.test.ts index d0486f1..abe0133 100644 --- a/packages/server/rpc/user/agent.integration.test.ts +++ b/packages/server/rpc/user/agent.integration.test.ts @@ -217,9 +217,10 @@ test("space.create provisions a space the caller owns + admins", async () => { ); expect(list.spaces.find((s) => s.id === res.id)?.admin).toBe(true); - // and the creator is owner of the root path + // the creator owns the shared root (and its home), not owner@root const ta = await coreStore(sql, coreSchema).buildTreeAccess(userId, res.id); - expect(ta).toContainEqual({ tree_path: ROOT_PATH, access: ACCESS.owner }); + expect(ta).toContainEqual({ tree_path: "share", access: ACCESS.owner }); + expect(ta).not.toContainEqual({ tree_path: ROOT_PATH, access: ACCESS.owner }); }); test("space.rename renames; space.delete removes the space + schema", async () => { diff --git a/packages/server/rpc/user/space.ts b/packages/server/rpc/user/space.ts index f98acaa..388284f 100644 --- a/packages/server/rpc/user/space.ts +++ b/packages/server/rpc/user/space.ts @@ -10,10 +10,8 @@ import { slugToSchema, } from "@memory.build/database"; import { - ACCESS, coreStore, type MemberSpace, - ROOT_PATH, type Space, } from "@memory.build/engine/core"; import type { @@ -34,6 +32,7 @@ import { spaceRenameParams, } from "@memory.build/protocol/user"; import type { Sql } from "postgres"; +import { addSpaceCreator } from "../../provision"; import { AppError } from "../errors"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; @@ -77,10 +76,12 @@ async function spaceList( } /** - * Create a new space and make the calling user its admin + owner of the root. - * Atomic: the core.space row, the me_ data schema, the membership, and the - * owner grant all land in one transaction (any failure rolls the schema back). - * The new space starts with an empty tree. + * Create a new space. The creator becomes a space admin and owner of its own + * home (via add_principal_to_space) and of the shared root (`share`) — but NOT + * owner@root, so it sees `/share` and `~` but not other members' homes. As an + * admin it can self-grant owner@root later if it wants the whole tree. Atomic: + * the core.space row, the me_ data schema, the membership, and the grant + * all land in one transaction (any failure rolls the schema back). */ async function spaceCreate( params: SpaceCreateParams, @@ -94,8 +95,7 @@ async function spaceCreate( const core = coreStore(tx as unknown as Sql, ctx.coreSchema); const spaceId = await core.createSpace(slug, params.name); await provisionSpace(tx, { slug }); // creates the me_ data schema - await core.addPrincipalToSpace(spaceId, ctx.userId, true); - await core.grantTreeAccess(spaceId, ctx.userId, ROOT_PATH, ACCESS.owner); + await addSpaceCreator(core, spaceId, ctx.userId); return spaceId; })) as string; From b474c3927804a1416466442864cdbad93e4bf3f3 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 14:09:52 +0200 Subject: [PATCH 085/156] feat(server): default memory.create tree to the shared root (share), not root memory.create / batchCreate now default a missing tree to SHARE_NAMESPACE ("share") rather than the space root, so a bare create lands in the shared area (which creators own) instead of root (which no regular member owns after the creator-grant change). Update the MCP create tool description; add a test that a default creator can bare-create into share. Co-Authored-By: Claude Opus 4.8 --- packages/cli/mcp/server.ts | 30 +++++++++---------- .../rpc/memory/memory.integration.test.ts | 9 ++++++ packages/server/rpc/memory/memory.ts | 7 +++-- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/packages/cli/mcp/server.ts b/packages/cli/mcp/server.ts index fffd19d..550055e 100644 --- a/packages/cli/mcp/server.ts +++ b/packages/cli/mcp/server.ts @@ -75,7 +75,7 @@ Docs: ${docUrl("me_memory_create")}`, .optional() .nullable() .describe( - "Hierarchical path (e.g., work.projects.me). Omit or null to store at the root.", + "Hierarchical path (e.g., share.work.projects; ~.* for your private home). Omit or null to store under the shared root (`share`).", ), temporal: z .object({ @@ -107,9 +107,9 @@ Docs: ${docUrl("me_memory_create")}`, tree: args.tree ?? undefined, temporal: args.temporal ? { - start: args.temporal.start, - end: args.temporal.end ?? undefined, - } + start: args.temporal.start, + end: args.temporal.end ?? undefined, + } : undefined, }); return { @@ -252,16 +252,16 @@ Docs: ${docUrl("me_memory_search")}`, tree: args.tree ?? undefined, temporal: args.temporal ? { - contains: args.temporal.contains ?? undefined, - overlaps: args.temporal.overlaps ?? undefined, - within: args.temporal.within ?? undefined, - } + contains: args.temporal.contains ?? undefined, + overlaps: args.temporal.overlaps ?? undefined, + within: args.temporal.within ?? undefined, + } : undefined, weights: args.weights ? { - fulltext: args.weights.fulltext ?? undefined, - semantic: args.weights.semantic ?? undefined, - } + fulltext: args.weights.fulltext ?? undefined, + semantic: args.weights.semantic ?? undefined, + } : undefined, candidateLimit: args.candidateLimit && args.candidateLimit > 0 @@ -367,9 +367,9 @@ Docs: ${docUrl("me_memory_update")}`, tree: args.tree ?? undefined, temporal: args.temporal ? { - start: args.temporal.start, - end: args.temporal.end ?? undefined, - } + start: args.temporal.start, + end: args.temporal.end ?? undefined, + } : undefined, }); return { @@ -817,7 +817,7 @@ Docs: ${docUrl("me_memory_export")}`, id: r.id, content: r.content, ...((r.meta as Record | undefined) && - Object.keys(r.meta as Record).length > 0 + Object.keys(r.meta as Record).length > 0 ? { meta: r.meta } : {}), ...(r.tree ? { tree: r.tree } : {}), diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index 3adb0b0..5252c84 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -170,6 +170,15 @@ test("~ home + lenient separators normalize on input, reverse-map on output", as ); }); +test("create without a tree defaults to the shared root (`share`)", async () => { + // the default creator owns `share` (no root grant in beforeEach), so a bare + // create lands there and succeeds. + const created = await call<{ tree: string }>("memory.create", { + content: "shared by default", + }); + expect(created.tree).toBe("share"); +}); + test("create → get round-trips content/tree/meta and createdBy is null", async () => { const created = await call<{ id: string; createdBy: string | null }>( "memory.create", diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 1f4b99d..5caf37f 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -7,8 +7,9 @@ * (the space model has no per-memory creator), search `total` is the returned * row count, and `orderBy` is ignored (ranked search is score-desc only). */ +import { SHARE_NAMESPACE } from "@memory.build/database"; import { generateEmbedding } from "@memory.build/embedding"; -import { ACCESS, ROOT_PATH } from "@memory.build/engine/core"; +import { ACCESS } from "@memory.build/engine/core"; import type { SearchResultItem, Memory as SpaceMemory, @@ -180,7 +181,7 @@ async function memoryCreate( id: params.id ?? undefined, content: params.content, meta: params.meta ?? undefined, - tree: inputTreePath(ctx, params.tree ?? ROOT_PATH), + tree: inputTreePath(ctx, params.tree ?? SHARE_NAMESPACE), temporal: formatTemporal(params.temporal), }), ); @@ -209,7 +210,7 @@ async function memoryBatchCreate( id: m.id ?? undefined, content: m.content, meta: m.meta ?? undefined, - tree: inputTreePath(ctx, m.tree ?? ROOT_PATH), + tree: inputTreePath(ctx, m.tree ?? SHARE_NAMESPACE), temporal: formatTemporal(m.temporal), }), ); From 554b2d340990db4e770a18ce41b4afc10741cc09 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 14:13:48 +0200 Subject: [PATCH 086/156] =?UTF-8?q?docs(claude):=20document=20the=20access?= =?UTF-8?q?=20model=20=E2=80=94=20home/share,=20creator=20grants,=20defaul?= =?UTF-8?q?t=20create?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reflect this session's changes in CLAUDE.md's authoritative summary: the structural (admin) vs data (owner@path) authority split; the home/share tree conventions and `~` sugar; a creator getting owner@home + owner@share (not owner@root); the `share` default for a bare memory.create; invite.* on the memory endpoint; and principal.list (admin) vs resolve/lookup (any member). Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a8ede73..b1448c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,9 +28,10 @@ Read the relevant docs before starting work on a subsystem. - **Schemas** (three, one database): `auth` (better-auth-shaped: `users`, `sessions`, `accounts`, `device_authorization`), `core` (control plane: `principal`, `space`, `principal_space`, `group_member`, `tree_access`, `api_key`), and per-space `me_` (data plane: the single `memory` table). `auth.users.id == core.principal.id` for user principals. - **Memory table** (per space): `content`, `meta` (JSONB), `tree` (ltree), `temporal` (tstzrange), `embedding` (halfvec(1536)). - **Search**: hybrid BM25 + semantic via Reciprocal Rank Fusion, computed in SQL functions. -- **Access**: no RLS. `core.build_tree_access(principalId, spaceId)` produces a `_tree_access` jsonb (rows of `tree_path` + `access`) passed into the space SQL functions (`search_memory`, `get_memory`, …). Three additive levels: **1 = read, 2 = write, 3 = owner**; owner@root (the empty ltree path) owns the whole space. +- **Access**: no RLS. `core.build_tree_access(principalId, spaceId)` produces a `_tree_access` jsonb (rows of `tree_path` + `access`) passed into the space SQL functions (`search_memory`, `get_memory`, …). Three additive levels: **1 = read, 2 = write, 3 = owner**; `owner@root` (the empty ltree path) owns the whole space, and an owner grant at any path delegates access-management within that subtree. Two axes: **structural** authority (`principal_space.admin` — roster mutations, groups, invitations) vs **data** authority (owner@path); an admin may also grant data and can self-grant `owner@root`. The auth gate is a non-empty `build_tree_access` (every member holds ≥1 grant). +- **Tree conventions**: two reserved roots — per-member `home.` (`~` is input sugar for it; a joining **user** is granted `owner@home`) and the shared `share`. A space **creator** gets `admin` + `owner@home` + `owner@share`, **not** `owner@root` — so it sees `share` and its own `~` but not other members' homes (as an admin it can self-grant `owner@root`). A bare `memory.create` (no `tree`) defaults to `share` (`SHARE_NAMESPACE`). - **API**: JSON-RPC 2.0 over HTTP, two endpoints: - - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `apiKey.*`). + - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `invite.*`, `apiKey.*`). - `/api/v1/user/rpc` — session only (an api key never authenticates here; agents can't manage agents). `whoami`, `agent.*`, `space.*`. - Plus REST OAuth device-flow endpoints under `/api/v1/auth/*`. - **Auth**: humans use a **session token** (OAuth device flow, GitHub/Google); agents use an **api key** (`me...`). Session + api-key secrets are sha256 (compared by equality in SQL), not argon2. @@ -42,7 +43,7 @@ Read the relevant docs before starting work on a subsystem. - **Principal** = the union **user | agent | group** (`principal.kind` = `'u'` | `'a'` | `'g'`). The space roster (`principal_space`) holds principals. `principal.member_id` is a generated column equal to `id` for users/agents (NOT groups). - **Member** = the **user/agent** sense only — group members and api-key holders. So params split as `principalId` (roster / grants, any kind) vs `memberId` (group membership, api keys; u|a only). The space-roster surface is principal-centric (`principal.*` methods, `SpacePrincipal` type), reserving "member" for u|a. - **Space**: identified by an immutable 12-char `slug` (which is the `me_` schema name, the api-key prefix, and the `X-Me-Space` value) and a renamable `name`. `me space rename` changes only the name. No org / engine / shard concepts. -- **Admin**: `principal_space.admin` is *structural* authority (manage groups + roster), distinct from data ownership (owner@root via `tree_access`). It transfers **transitively** through a group whose own `principal_space.admin` is true; agents are never admins. +- **Admin**: `principal_space.admin` is *structural* authority — roster mutations (`principal.add`/`remove`), groups, and invitations (`invite.*`) — distinct from data ownership (owner@path via `tree_access`). Enumerating the whole roster (`principal.list`) is admin-only; **any member** may `principal.resolve`/`lookup` (a targeted name↔id lookup, not enumeration). Admin transfers **transitively** through a group whose own `principal_space.admin` is true; agents are never admins. - **Transitive membership** (Model 2): a group member gains the group's space membership, its space-admin (if the group is admin), and its tree-access grants. ## Project Structure @@ -156,7 +157,7 @@ create table me.memory - **One DB, one pool**: `auth` + `core` + every `me_` live in one Postgres database behind one postgres.js pool (plus a dedicated worker pool). Sharding / pgdog distribution is deferred; the per-slug schema model keeps a future re-split cheap. - **Single memory table per space**: all memory lives in `me_.memory`. Complexity comes from conventions in `meta` and `tree`, not schema proliferation. - **Database-native**: PostgreSQL extensions (ltree, pgvector/halfvec, JSONB GIN, tstzrange, BM25) instead of application-layer abstractions. -- **Access via `tree_access`, not RLS**: RLS was unperformant. `build_tree_access` produces a `_tree_access` jsonb passed into the space functions; there is no `me.user_id` GUC. Three levels (read/write/owner); owner@root delegates within a subtree. +- **Access via `tree_access`, not RLS**: RLS was unperformant. `build_tree_access` produces a `_tree_access` jsonb passed into the space functions; there is no `me.user_id` GUC. Three levels (read/write/owner); an owner grant delegates access-management within its subtree (owner@root = the whole space). - **Two endpoints, two auth modes**: memory RPC (session or api key + `X-Me-Space`) vs user RPC (session only). `extractBearerToken` is the one shared auth helper. - **Principal vs member** terminology (see above): principal = u|a|g; member/`memberId` = u|a. - **CLI credentials**: the credentials file (`~/.config/me/credentials.yaml`, 0600) stores only the **session token + active space** per server. **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE`. *TODO*: move the session token into the OS keychain with a 0600-file fallback. From 09fb4af8bb023d70d0b0bde12eeefddf6247ee2a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 14:26:42 +0200 Subject: [PATCH 087/156] refactor(server): drop the space-manager concept from grant authority MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grant.list defaults its ownership check to root when no tree filter is given (owner@root authorizes the whole space via ownsTreePath); grant.set/remove check ctx.admin directly and let owner@root fall through to requireTreeOwner (which owns every path). Both now rest on requireSpaceAdmin / tree-ownership instead of requireSpaceManager (admin-or-owner@root) — which, with isSpaceManager and isSpaceOwner, is removed as dead. Behavior unchanged. Co-Authored-By: Claude Opus 4.8 --- packages/server/rpc/memory/grant.ts | 26 +++++++++++++------------- packages/server/rpc/memory/support.ts | 26 -------------------------- 2 files changed, 13 insertions(+), 39 deletions(-) diff --git a/packages/server/rpc/memory/grant.ts b/packages/server/rpc/memory/grant.ts index 60fe517..8f9af81 100644 --- a/packages/server/rpc/memory/grant.ts +++ b/packages/server/rpc/memory/grant.ts @@ -2,6 +2,7 @@ * Tree-access grant handlers (grant.*). Three additive levels * (1 = read, 2 = write, 3 = owner); owner listing is grant.list filtered to 3. */ +import { ROOT_PATH } from "@memory.build/engine/core"; import type { GrantListParams, GrantListResult, @@ -21,9 +22,8 @@ import { callerOwnsAgent, guardCore, inputTreePath, - isSpaceManager, ownsTreePath, - requireSpaceManager, + requireSpaceAdmin, requireTreeOwner, toTreeGrantResponse, } from "./support"; @@ -32,8 +32,8 @@ import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; /** * Authority to grant/remove access at a path. Allowed when any of: * - the target is the caller's OWN agent (self-service — capped anyway); - * - the caller is a space admin / owner; - * - the caller owns the tree path (owning a subtree delegates control within it). + * - the caller is a space admin (admins manage all access); + * - the caller owns the path or an ancestor (owner@root owns the whole tree). */ async function requireGrantAuthority( ctx: SpaceRpcContext, @@ -41,7 +41,7 @@ async function requireGrantAuthority( treePath: string, ): Promise { if (await callerOwnsAgent(ctx, principalId)) return; - if (isSpaceManager(ctx)) return; + if (ctx.admin) return; requireTreeOwner(ctx, treePath); } @@ -84,24 +84,24 @@ async function grantList( ): Promise { assertSpaceRpcContext(context); const ctx = context as SpaceRpcContext; - const treePath = + // No path filter means the whole space, i.e. the root path. Listing grants + // under a path requires owning that path (root → owning the whole space), + // else space-admin. Listing your OWN agent's grants is always self-service. + const under = params.treePath !== undefined && params.treePath !== null ? inputTreePath(ctx, params.treePath) - : undefined; - // Authorized when listing your OWN agent's grants, or a subtree you own, or - // (broadly) as a space manager. + : ROOT_PATH; const ownAgent = params.principalId !== undefined && params.principalId !== null && (await callerOwnsAgent(ctx, params.principalId)); - const pathOwner = treePath !== undefined && ownsTreePath(ctx, treePath); - if (!ownAgent && !pathOwner) { - requireSpaceManager(ctx); + if (!ownAgent && !ownsTreePath(ctx, under)) { + requireSpaceAdmin(ctx); } const grants = await ctx.core.listTreeAccessGrants( ctx.space.id, params.principalId ?? undefined, - treePath, + under, ); return { grants: grants.map((g) => toTreeGrantResponse(g, ctx)) }; } diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts index 0df1d9e..042b9a4 100644 --- a/packages/server/rpc/memory/support.ts +++ b/packages/server/rpc/memory/support.ts @@ -74,13 +74,6 @@ function asValidationError(e: unknown): AppError { : new AppError("VALIDATION_ERROR", "Invalid tree path"); } -/** Owner-level grant (3) at the space root — owns the whole space. */ -export function isSpaceOwner(context: SpaceRpcContext): boolean { - return context.treeAccess.some( - (g) => g.tree_path === ROOT_PATH && g.access >= ACCESS.owner, - ); -} - /** * Structural authority over the space (principal_space.admin). Required for * managing groups — a structural construct of the space, distinct from data @@ -115,25 +108,6 @@ export async function requireGroupAdmin( } } -/** - * Space-management authority: a space admin (principal_space.admin) or the - * space owner (owner@root). Gates roster management and broad grant listing — - * controlling access to data the owner owns. (Per-subtree grant delegation is - * handled by requireTreeOwner; group structure requires requireSpaceAdmin.) - */ -export function isSpaceManager(context: SpaceRpcContext): boolean { - return context.admin || isSpaceOwner(context); -} - -export function requireSpaceManager(context: SpaceRpcContext): void { - if (!isSpaceManager(context)) { - throw new AppError( - "FORBIDDEN", - "Space management requires being a space admin or owner", - ); - } -} - /** True if `ancestor` is an ancestor-or-self of `path` (ltree `@>`). */ function isAncestorOrSelf(ancestor: string, path: string): boolean { return ( From d13ebc3a4b897f6b2119475c6ce6c33c2003eb4d Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 14:26:50 +0200 Subject: [PATCH 088/156] style(mcp): biome formatting (ternary indentation in server.ts) Pure formatting normalization that a biome --write applied after b474c39 committed the file unformatted; no behavior change. Co-Authored-By: Claude Opus 4.8 --- packages/cli/mcp/server.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/cli/mcp/server.ts b/packages/cli/mcp/server.ts index 550055e..8be8e93 100644 --- a/packages/cli/mcp/server.ts +++ b/packages/cli/mcp/server.ts @@ -107,9 +107,9 @@ Docs: ${docUrl("me_memory_create")}`, tree: args.tree ?? undefined, temporal: args.temporal ? { - start: args.temporal.start, - end: args.temporal.end ?? undefined, - } + start: args.temporal.start, + end: args.temporal.end ?? undefined, + } : undefined, }); return { @@ -252,16 +252,16 @@ Docs: ${docUrl("me_memory_search")}`, tree: args.tree ?? undefined, temporal: args.temporal ? { - contains: args.temporal.contains ?? undefined, - overlaps: args.temporal.overlaps ?? undefined, - within: args.temporal.within ?? undefined, - } + contains: args.temporal.contains ?? undefined, + overlaps: args.temporal.overlaps ?? undefined, + within: args.temporal.within ?? undefined, + } : undefined, weights: args.weights ? { - fulltext: args.weights.fulltext ?? undefined, - semantic: args.weights.semantic ?? undefined, - } + fulltext: args.weights.fulltext ?? undefined, + semantic: args.weights.semantic ?? undefined, + } : undefined, candidateLimit: args.candidateLimit && args.candidateLimit > 0 @@ -367,9 +367,9 @@ Docs: ${docUrl("me_memory_update")}`, tree: args.tree ?? undefined, temporal: args.temporal ? { - start: args.temporal.start, - end: args.temporal.end ?? undefined, - } + start: args.temporal.start, + end: args.temporal.end ?? undefined, + } : undefined, }); return { @@ -817,7 +817,7 @@ Docs: ${docUrl("me_memory_export")}`, id: r.id, content: r.content, ...((r.meta as Record | undefined) && - Object.keys(r.meta as Record).length > 0 + Object.keys(r.meta as Record).length > 0 ? { meta: r.meta } : {}), ...(r.tree ? { tree: r.tree } : {}), From 8543a555a1d3a581fcae4ae860d2f20de152d2a3 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 14:58:53 +0200 Subject: [PATCH 089/156] feat(cli): store the session token in the OS keychain with a file fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add keychain.ts — macOS `security` + Linux libsecret `secret-tool` backends (shelled out, so the compiled binary needs no native module), with ME_NO_KEYCHAIN to force off. credentials.ts keeps the session token in the keychain when available (dropping any file copy) and uses the 0600 file otherwise; resolve reads env -> file -> keychain, clear cleans both. Windows / headless-without-a- secret-service use the file. Tests cover the file-fallback and disabled paths; a live keychain round-trip runs wherever a keychain is usable (skips on CI / Linux-without-libsecret / locked), verified on macOS. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 2 +- TODO.md | 12 -- packages/cli/credentials.test.ts | 101 +++++++++++++++++ packages/cli/credentials.ts | 61 +++++----- packages/cli/keychain.test.ts | 59 ++++++++++ packages/cli/keychain.ts | 189 +++++++++++++++++++++++++++++++ 6 files changed, 385 insertions(+), 39 deletions(-) create mode 100644 packages/cli/credentials.test.ts create mode 100644 packages/cli/keychain.test.ts create mode 100644 packages/cli/keychain.ts diff --git a/CLAUDE.md b/CLAUDE.md index b1448c3..c6587d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -160,7 +160,7 @@ create table me.memory - **Access via `tree_access`, not RLS**: RLS was unperformant. `build_tree_access` produces a `_tree_access` jsonb passed into the space functions; there is no `me.user_id` GUC. Three levels (read/write/owner); an owner grant delegates access-management within its subtree (owner@root = the whole space). - **Two endpoints, two auth modes**: memory RPC (session or api key + `X-Me-Space`) vs user RPC (session only). `extractBearerToken` is the one shared auth helper. - **Principal vs member** terminology (see above): principal = u|a|g; member/`memberId` = u|a. -- **CLI credentials**: the credentials file (`~/.config/me/credentials.yaml`, 0600) stores only the **session token + active space** per server. **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE`. *TODO*: move the session token into the OS keychain with a 0600-file fallback. +- **CLI credentials**: the **session token** lives in the OS keychain when available (macOS `security`, Linux `secret-tool` via libsecret; `ME_NO_KEYCHAIN=1` forces off), otherwise in the 0600 credentials file (`~/.config/me/credentials.yaml`), which always holds the non-secret per-server **active space** + default server. **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE` / `ME_SESSION_TOKEN` / `ME_NO_KEYCHAIN`. - **Header constants** (`CLIENT_VERSION_HEADER`, `SPACE_HEADER`) live in `@memory.build/protocol/headers`. - **MCP compatibility**: all tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). diff --git a/TODO.md b/TODO.md index 6716a34..907de5f 100644 --- a/TODO.md +++ b/TODO.md @@ -175,18 +175,6 @@ form — the right convention is what's natural for users, not what ltree accept unit + ghost db suites pass. (bootstrap's lock moved from a hardcoded single-key id to the shared two-key derived lock.) -## OS keychain for CLI credentials - -The CLI credentials file (`~/.config/me/credentials.yaml`, 0600) stores the -session token + active space in plaintext. (Api keys are never stored — they -come from `ME_API_KEY` only.) A code TODO marker lives in -`packages/cli/credentials.ts`. - -- [ ] Move the session token into the OS keychain (macOS `security`, Linux - `secret-tool`, Windows credential manager) with a fallback to the 0600 - file when no keychain is available (CI, headless Linux). The file would - then hold only non-secret pointers (`default_server`, `active_space`). - ## Refresh `docs/` for the principal / space model The `docs/` pages (getting-started, concepts, access-control, `cli/*`, `mcp/*`) diff --git a/packages/cli/credentials.test.ts b/packages/cli/credentials.test.ts new file mode 100644 index 0000000..9adc2be --- /dev/null +++ b/packages/cli/credentials.test.ts @@ -0,0 +1,101 @@ +/** + * Credential storage tests — the file-fallback path. + * + * Forces the 0600-file fallback (ME_NO_KEYCHAIN) and an isolated XDG config dir + * so the behavior is deterministic across platforms. The OS keychain backend is + * exercised separately in keychain.test.ts. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as creds from "./credentials.ts"; +import { resetKeychainForTests } from "./keychain.ts"; + +const SERVER = "https://api.example.com"; +const TOKEN_ENVS = ["ME_SESSION_TOKEN", "ME_SPACE", "ME_SERVER", "ME_API_KEY"]; +// Every env key these tests touch — snapshotted and restored so the ambient +// environment (and other test files in the same process) is left untouched. +const ENV_KEYS = [...TOKEN_ENVS, "XDG_CONFIG_HOME", "ME_NO_KEYCHAIN"]; + +let configDir: string; +let savedEnv: Record; + +beforeEach(() => { + savedEnv = {}; + for (const k of ENV_KEYS) savedEnv[k] = process.env[k]; + + configDir = mkdtempSync(join(tmpdir(), "me-creds-")); + process.env.XDG_CONFIG_HOME = configDir; + process.env.ME_NO_KEYCHAIN = "1"; // force the file fallback + for (const k of TOKEN_ENVS) delete process.env[k]; + resetKeychainForTests(); +}); + +afterEach(() => { + rmSync(configDir, { recursive: true, force: true }); + for (const k of ENV_KEYS) { + const v = savedEnv[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + resetKeychainForTests(); +}); + +test("store + resolve a session token (file fallback)", () => { + creds.storeSessionToken(SERVER, "tok-123"); + const r = creds.resolveCredentials(SERVER); + expect(r.server).toBe(SERVER); + expect(r.sessionToken).toBe("tok-123"); + // fallback stores the token in the file (only when there's no keychain) + expect(creds.getServerCredentials(SERVER).session_token).toBe("tok-123"); +}); + +test("the credentials file is written 0600", () => { + creds.storeSessionToken(SERVER, "tok"); + const file = join(configDir, "me", "credentials.yaml"); + expect(existsSync(file)).toBe(true); + // low 9 permission bits = rw------- (0o600) + expect(statSync(file).mode & 0o777).toBe(0o600); + // sanity: the token is actually in the file in fallback mode + expect(readFileSync(file, "utf-8")).toContain("tok"); +}); + +test("clearSessionToken removes the token", () => { + creds.storeSessionToken(SERVER, "tok-123"); + creds.clearSessionToken(SERVER); + expect(creds.resolveCredentials(SERVER).sessionToken).toBeUndefined(); +}); + +test("ME_SESSION_TOKEN env overrides the stored token", () => { + creds.storeSessionToken(SERVER, "stored"); + process.env.ME_SESSION_TOKEN = "from-env"; + expect(creds.resolveCredentials(SERVER).sessionToken).toBe("from-env"); +}); + +test("active space: set / resolve / clear; ME_SPACE wins", () => { + creds.setActiveSpace(SERVER, "abc123def456"); + expect(creds.resolveCredentials(SERVER).activeSpace).toBe("abc123def456"); + + process.env.ME_SPACE = "envspace0001"; + expect(creds.resolveCredentials(SERVER).activeSpace).toBe("envspace0001"); + delete process.env.ME_SPACE; + + creds.clearActiveSpace(SERVER); + expect(creds.resolveCredentials(SERVER).activeSpace).toBeUndefined(); +}); + +test("clearServerCredentials drops the entry and resets the default", () => { + creds.storeSessionToken(SERVER, "tok"); + creds.setActiveSpace(SERVER, "abc123def456"); + creds.clearServerCredentials(SERVER); + const r = creds.resolveCredentials(SERVER); + expect(r.sessionToken).toBeUndefined(); + expect(r.activeSpace).toBeUndefined(); +}); diff --git a/packages/cli/credentials.ts b/packages/cli/credentials.ts index 835f743..74877cd 100644 --- a/packages/cli/credentials.ts +++ b/packages/cli/credentials.ts @@ -1,32 +1,28 @@ /** - * Credential storage — multi-server, multi-space credential management. + * Credential storage — multi-server credential management. * - * Stores the session token (humans) and per-space agent API keys in - * $XDG_CONFIG_HOME/me/credentials.yaml (default: ~/.config/me/). - * - * The file holds the human session token and the active space; it never stores - * api keys. Api keys are for agents, which run elsewhere and receive their key - * via the `ME_API_KEY` env var (or pasted into their MCP config). `apiKey.create` - * prints the key once — the operator places it where the agent runs. + * The session token (the one persisted secret) lives in the OS keychain when + * available (see ./keychain.ts), else in the 0600 credentials file at + * $XDG_CONFIG_HOME/me/credentials.yaml (default: ~/.config/me/). The file always + * holds the non-secret pointers (default_server, per-server active_space) and — + * only on hosts without a keychain — the session token too. Api keys are never + * stored: agents receive their key via `ME_API_KEY` (or their MCP config); + * `apiKey.create` prints it once. * * File format: * ```yaml * default_server: https://api.memory.build * servers: * https://api.memory.build: - * session_token: "..." # human session (used with X-Me-Space) + * session_token: "..." # only on hosts without a keychain * active_space: "abc123def456" # active space slug (the X-Me-Space) * ``` - * - * TODO(keychain): move the session token into the OS keychain (macOS `security`, - * Linux `secret-tool`, Windows credential manager) with a fall back to this 0600 - * file when no keychain is available (CI, headless Linux). The file would then - * hold only non-secret pointers (default_server, active_space). */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { parse, stringify } from "yaml"; +import { keychainDelete, keychainGet, keychainSet } from "./keychain.ts"; // ============================================================================= // Constants @@ -182,8 +178,10 @@ export function getServerCredentials(server: string): ServerCredentials { } /** - * Store a session token for a server. - * Also sets this server as the default. + * Store a session token for a server, and set it as the default. Prefers the OS + * keychain; only when that's unavailable does the token land in the (0600) file. + * Either way the file records the server (default_server) so the token can be + * found again — and any stale file copy is dropped once the keychain has it. */ export function storeSessionToken(server: string, token: string): void { const creds = readCredentials(); @@ -192,7 +190,12 @@ export function storeSessionToken(server: string, token: string): void { if (!creds.servers[origin]) { creds.servers[origin] = {}; } - creds.servers[origin].session_token = token; + if (keychainSet(origin, token)) { + // Keychain is the source of truth now — don't also persist it in the file. + delete creds.servers[origin].session_token; + } else { + creds.servers[origin].session_token = token; + } creds.default_server = origin; writeCredentials(creds); @@ -249,29 +252,29 @@ export function resolveSpace( * API keys in place. Used after the server tells us the session is expired so * the next command surfaces "Not logged in" instead of repeating the 401. * - * No-op if no credentials are stored for the server, or if the token came + * Clears both the keychain entry and any file copy. No-op for a token that came * from $ME_SESSION_TOKEN (we can't unset an env var the user controls). */ export function clearSessionToken(server: string): void { - const creds = readCredentials(); const origin = normalizeOrigin(server); + keychainDelete(origin); + const creds = readCredentials(); const entry = creds.servers[origin]; - if (!entry?.session_token) { - return; + if (entry?.session_token) { + delete entry.session_token; + writeCredentials(creds); } - delete entry.session_token; - - writeCredentials(creds); } /** - * Clear all credentials for a server. + * Clear all credentials for a server (keychain token + file entry). */ export function clearServerCredentials(server: string): void { const creds = readCredentials(); const origin = normalizeOrigin(server); + keychainDelete(origin); delete creds.servers[origin]; // If we just cleared the default server, reset to default @@ -312,7 +315,13 @@ export function resolveCredentials(serverFlag?: string): ResolvedCredentials { return { server, - sessionToken: process.env.ME_SESSION_TOKEN ?? stored.session_token, + // env wins; then the file (the keychain-free fallback); then the keychain. + // The token lives in exactly one of file/keychain, so checking the file + // first avoids a keychain lookup on hosts that use the file fallback. + sessionToken: + process.env.ME_SESSION_TOKEN ?? + stored.session_token ?? + keychainGet(normalizeOrigin(server)), apiKey: process.env.ME_API_KEY, activeSpace: process.env.ME_SPACE ?? stored.active_space, }; diff --git a/packages/cli/keychain.test.ts b/packages/cli/keychain.test.ts new file mode 100644 index 0000000..5df81f8 --- /dev/null +++ b/packages/cli/keychain.test.ts @@ -0,0 +1,59 @@ +/** + * Keychain backend tests. + * + * The disabled path (ME_NO_KEYCHAIN) is deterministic everywhere. The live + * round-trip touches the real OS keychain, so it runs only where one is usable: + * it skips gracefully on Linux without libsecret, in CI, with a locked store, or + * when ME_NO_KEYCHAIN is set — but on an interactive mac, where a keychain + * should always work, a failure is treated as a real regression. + */ +import { afterEach, expect, test } from "bun:test"; +import { + keychainAvailable, + keychainDelete, + keychainGet, + keychainSet, + resetKeychainForTests, +} from "./keychain.ts"; + +// The ambient value, restored after each test so a dev/CI ME_NO_KEYCHAIN survives. +const AMBIENT_NO_KEYCHAIN = process.env.ME_NO_KEYCHAIN; + +afterEach(() => { + if (AMBIENT_NO_KEYCHAIN === undefined) delete process.env.ME_NO_KEYCHAIN; + else process.env.ME_NO_KEYCHAIN = AMBIENT_NO_KEYCHAIN; + resetKeychainForTests(); +}); + +test("ME_NO_KEYCHAIN forces the file fallback", () => { + process.env.ME_NO_KEYCHAIN = "1"; + resetKeychainForTests(); + expect(keychainAvailable()).toBe(false); + expect(keychainSet("acct", "secret")).toBe(false); + expect(keychainGet("acct")).toBeUndefined(); + keychainDelete("acct"); // no-op, must not throw +}); + +test("keychain round-trip when an OS keychain is usable", () => { + // Respect an ambient opt-out — nothing to exercise. + const v = process.env.ME_NO_KEYCHAIN; + if (v === "1" || v === "true") return; + resetKeychainForTests(); + + const account = `https://kc-test-${crypto.randomUUID()}.example.com`; + if (!keychainSet(account, "live-secret")) { + // No usable keychain (Linux without secret-tool, CI, locked store). Skip — + // but an interactive mac should always have one, so a miss there is a bug. + expect(process.platform === "darwin" && !process.env.CI).toBe(false); + return; + } + + try { + expect(keychainGet(account)).toBe("live-secret"); + keychainSet(account, "updated-secret"); // -U updates in place + expect(keychainGet(account)).toBe("updated-secret"); + } finally { + keychainDelete(account); + } + expect(keychainGet(account)).toBeUndefined(); +}); diff --git a/packages/cli/keychain.ts b/packages/cli/keychain.ts new file mode 100644 index 0000000..84bfd48 --- /dev/null +++ b/packages/cli/keychain.ts @@ -0,0 +1,189 @@ +/** + * OS keychain for the CLI session token, with a 0600-file fallback. + * + * The session token is the only secret the CLI persists. When an OS secret + * store is available we keep it there (one entry per server origin); otherwise + * the caller falls back to the 0600 credentials file. Backends shell out to the + * platform tool, so the compiled `me` binary needs no native module: + * + * - macOS: `security` (the login keychain) + * - Linux: `secret-tool` (libsecret / the Secret Service) + * + * Anything else (Windows, headless Linux without a Secret Service) reports + * unavailable and the caller uses the file. Set `ME_NO_KEYCHAIN=1` to force the + * file fallback everywhere (CI, debugging, sandboxes). + * + * Detection + operations are best-effort and defensive: a missing tool, a + * non-running secret service, a locked store, or a spawn error is treated as + * "not stored / not found" so the file fallback transparently kicks in. The + * `account` is the (normalized) server origin; the secret is the session token. + */ + +/** Keychain service name — how the entries appear in Keychain Access / seahorse. */ +const SERVICE = "memory.build"; + +/** A spawn timeout so a prompting/locked secret store can't hang the CLI. */ +const SPAWN_TIMEOUT_MS = 5_000; + +interface Backend { + get(account: string): string | undefined; + set(account: string, secret: string): boolean; + del(account: string): void; +} + +function keychainDisabled(): boolean { + const v = process.env.ME_NO_KEYCHAIN; + return v === "1" || v === "true"; +} + +/** Run a command, capturing stdout; returns null on spawn failure. */ +function run( + cmd: string[], + stdin?: string, +): { exitCode: number; stdout: string } | null { + try { + const r = Bun.spawnSync({ + cmd, + stdin: stdin !== undefined ? new TextEncoder().encode(stdin) : undefined, + stdout: "pipe", + stderr: "pipe", + timeout: SPAWN_TIMEOUT_MS, + }); + return { exitCode: r.exitCode ?? 1, stdout: r.stdout.toString() }; + } catch { + return null; + } +} + +// macOS — the `security` CLI against the login keychain. The secret is passed +// via argv (-w); it is briefly visible to `ps`, but only to the same user, who +// can already read the 0600 fallback file. +const darwinBackend: Backend = { + get(account) { + const r = run([ + "security", + "find-generic-password", + "-s", + SERVICE, + "-a", + account, + "-w", + ]); + if (!r || r.exitCode !== 0) return undefined; + const out = r.stdout.replace(/\n$/, ""); + return out.length > 0 ? out : undefined; + }, + set(account, secret) { + const r = run([ + "security", + "add-generic-password", + "-s", + SERVICE, + "-a", + account, + "-w", + secret, + "-U", // update the entry if it already exists + ]); + return r?.exitCode === 0; + }, + del(account) { + run(["security", "delete-generic-password", "-s", SERVICE, "-a", account]); + }, +}; + +// Linux — libsecret's `secret-tool` (Secret Service). The secret is read from +// stdin (never argv). `lookup` prints the secret with no trailing newline. +const linuxBackend: Backend = { + get(account) { + const r = run([ + "secret-tool", + "lookup", + "service", + SERVICE, + "account", + account, + ]); + if (!r || r.exitCode !== 0) return undefined; + return r.stdout.length > 0 ? r.stdout : undefined; + }, + set(account, secret) { + const r = run( + [ + "secret-tool", + "store", + "--label=memory.build CLI session", + "service", + SERVICE, + "account", + account, + ], + secret, + ); + return r?.exitCode === 0; + }, + del(account) { + run(["secret-tool", "clear", "service", SERVICE, "account", account]); + }, +}; + +let resolved: Backend | null | undefined; + +/** The backend for this host, or null when no keychain is usable. Memoized. */ +function backend(): Backend | null { + if (resolved !== undefined) return resolved; + resolved = selectBackend(); + return resolved; +} + +function selectBackend(): Backend | null { + if (keychainDisabled()) return null; + if (process.platform === "darwin") return darwinBackend; + if (process.platform === "linux" && Bun.which("secret-tool")) { + return linuxBackend; + } + return null; +} + +/** Reset the memoized backend — for tests that toggle `ME_NO_KEYCHAIN`. */ +export function resetKeychainForTests(): void { + resolved = undefined; +} + +/** True if an OS keychain backend is available on this host. */ +export function keychainAvailable(): boolean { + return backend() !== null; +} + +/** Read the secret for `account`, or undefined if absent / unavailable. */ +export function keychainGet(account: string): string | undefined { + const b = backend(); + if (!b) return undefined; + try { + return b.get(account); + } catch { + return undefined; + } +} + +/** Store the secret for `account`. Returns true iff it landed in the keychain. */ +export function keychainSet(account: string, secret: string): boolean { + const b = backend(); + if (!b) return false; + try { + return b.set(account, secret); + } catch { + return false; + } +} + +/** Remove the secret for `account` (no-op if absent / unavailable). */ +export function keychainDelete(account: string): void { + const b = backend(); + if (!b) return; + try { + b.del(account); + } catch { + // best-effort + } +} From 166fd4d30acdbbad66b604fc95089df41c5c2f6a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 15:26:55 +0200 Subject: [PATCH 090/156] refactor(cli): split non-secret config into config.yaml; credentials.yaml is secret-only Move default_server + per-server active_space to config.yaml (non-secret); credentials.yaml now holds only the session-token fallback (0600), empty/absent on keychain hosts. A pre-split credentials.yaml is migrated on first read, so existing logins keep their token/server/space. me logout now clears just the session secret and keeps the non-secret config (re-login resumes). resolve reads the space from config and the token from env -> file -> keychain. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 2 +- packages/cli/credentials.test.ts | 62 +++++- packages/cli/credentials.ts | 367 +++++++++++++++++-------------- 3 files changed, 261 insertions(+), 170 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c6587d3..c2ee9c4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -160,7 +160,7 @@ create table me.memory - **Access via `tree_access`, not RLS**: RLS was unperformant. `build_tree_access` produces a `_tree_access` jsonb passed into the space functions; there is no `me.user_id` GUC. Three levels (read/write/owner); an owner grant delegates access-management within its subtree (owner@root = the whole space). - **Two endpoints, two auth modes**: memory RPC (session or api key + `X-Me-Space`) vs user RPC (session only). `extractBearerToken` is the one shared auth helper. - **Principal vs member** terminology (see above): principal = u|a|g; member/`memberId` = u|a. -- **CLI credentials**: the **session token** lives in the OS keychain when available (macOS `security`, Linux `secret-tool` via libsecret; `ME_NO_KEYCHAIN=1` forces off), otherwise in the 0600 credentials file (`~/.config/me/credentials.yaml`), which always holds the non-secret per-server **active space** + default server. **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE` / `ME_SESSION_TOKEN` / `ME_NO_KEYCHAIN`. +- **CLI credentials**: split across `~/.config/me/` — **`config.yaml`** (non-secret: default server + per-server **active space** / the X-Me-Space) and **`credentials.yaml`** (0600, secret session-token *fallback* only). The **session token** lives in the OS keychain when available (macOS `security`, Linux `secret-tool` via libsecret; `ME_NO_KEYCHAIN=1` forces off), else in `credentials.yaml` (empty/absent on keychain hosts); a pre-split `credentials.yaml` is migrated on first read. `me logout` clears the session secret but keeps the non-secret config (so re-login resumes). **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE` / `ME_SESSION_TOKEN` / `ME_NO_KEYCHAIN`. - **Header constants** (`CLIENT_VERSION_HEADER`, `SPACE_HEADER`) live in `@memory.build/protocol/headers`. - **MCP compatibility**: all tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). diff --git a/packages/cli/credentials.test.ts b/packages/cli/credentials.test.ts index 9adc2be..1a19e3a 100644 --- a/packages/cli/credentials.test.ts +++ b/packages/cli/credentials.test.ts @@ -8,10 +8,12 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { existsSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, + writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -53,8 +55,8 @@ test("store + resolve a session token (file fallback)", () => { const r = creds.resolveCredentials(SERVER); expect(r.server).toBe(SERVER); expect(r.sessionToken).toBe("tok-123"); - // fallback stores the token in the file (only when there's no keychain) - expect(creds.getServerCredentials(SERVER).session_token).toBe("tok-123"); + // fallback stores the token in the secrets file (no keychain) + expect(creds.getServerSecrets(SERVER).session_token).toBe("tok-123"); }); test("the credentials file is written 0600", () => { @@ -91,11 +93,61 @@ test("active space: set / resolve / clear; ME_SPACE wins", () => { expect(creds.resolveCredentials(SERVER).activeSpace).toBeUndefined(); }); -test("clearServerCredentials drops the entry and resets the default", () => { +test("logout clears the secret but keeps the active space", () => { creds.storeSessionToken(SERVER, "tok"); creds.setActiveSpace(SERVER, "abc123def456"); - creds.clearServerCredentials(SERVER); + creds.clearServerCredentials(SERVER); // logout const r = creds.resolveCredentials(SERVER); expect(r.sessionToken).toBeUndefined(); - expect(r.activeSpace).toBeUndefined(); + expect(r.activeSpace).toBe("abc123def456"); // non-secret config survives logout +}); + +test("secrets and config live in separate files", () => { + creds.storeSessionToken(SERVER, "tok-sep"); + creds.setActiveSpace(SERVER, "abc123def456"); + const configFile = readFileSync( + join(configDir, "me", "config.yaml"), + "utf-8", + ); + const credsFile = readFileSync( + join(configDir, "me", "credentials.yaml"), + "utf-8", + ); + // config.yaml has the active space (non-secret), not the token + expect(configFile).toContain("abc123def456"); + expect(configFile).not.toContain("tok-sep"); + // credentials.yaml has the token (fallback), not the active space + expect(credsFile).toContain("tok-sep"); + expect(credsFile).not.toContain("abc123def456"); +}); + +test("migrates a legacy credentials.yaml (token + active_space + default)", () => { + // a pre-split credentials.yaml that bundled everything together + const dir = join(configDir, "me"); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + writeFileSync( + join(dir, "credentials.yaml"), + [ + `default_server: ${SERVER}`, + "servers:", + ` ${SERVER}:`, + " session_token: legacy-tok", + " active_space: legacyspace1", + ].join("\n"), + { mode: 0o600 }, + ); + + // reading resolves all three, migrating the non-secret bits out + const r = creds.resolveCredentials(); + expect(r.server).toBe(SERVER); + expect(r.sessionToken).toBe("legacy-tok"); + expect(r.activeSpace).toBe("legacyspace1"); + + // config.yaml now exists with the non-secret bits; credentials.yaml is + // secret-only (no active_space left behind) + const configFile = readFileSync(join(dir, "config.yaml"), "utf-8"); + expect(configFile).toContain("legacyspace1"); + const credsFile = readFileSync(join(dir, "credentials.yaml"), "utf-8"); + expect(credsFile).toContain("legacy-tok"); + expect(credsFile).not.toContain("legacyspace1"); }); diff --git a/packages/cli/credentials.ts b/packages/cli/credentials.ts index 74877cd..cef8e13 100644 --- a/packages/cli/credentials.ts +++ b/packages/cli/credentials.ts @@ -1,22 +1,33 @@ /** - * Credential storage — multi-server credential management. + * Credential + config storage — multi-server. * - * The session token (the one persisted secret) lives in the OS keychain when - * available (see ./keychain.ts), else in the 0600 credentials file at - * $XDG_CONFIG_HOME/me/credentials.yaml (default: ~/.config/me/). The file always - * holds the non-secret pointers (default_server, per-server active_space) and — - * only on hosts without a keychain — the session token too. Api keys are never - * stored: agents receive their key via `ME_API_KEY` (or their MCP config); - * `apiKey.create` prints it once. + * Two files under $XDG_CONFIG_HOME/me (default ~/.config/me): + * - config.yaml — non-secret: the default server + each server's active + * space (the X-Me-Space). + * - credentials.yaml — 0600, secrets only: the session-token fallback, used + * when no OS keychain is available (see ./keychain.ts); + * empty / absent on hosts with a keychain. * - * File format: + * The session token (the one secret) prefers the OS keychain; the file is the + * fallback. Api keys are never stored — agents get their key via `ME_API_KEY` + * (or their MCP config); `apiKey.create` prints it once. + * + * config.yaml: * ```yaml * default_server: https://api.memory.build * servers: * https://api.memory.build: - * session_token: "..." # only on hosts without a keychain - * active_space: "abc123def456" # active space slug (the X-Me-Space) + * active_space: abc123def456 * ``` + * credentials.yaml (0600): + * ```yaml + * servers: + * https://api.memory.build: + * session_token: "..." # only when there's no keychain + * ``` + * + * A pre-split credentials.yaml (which once held default_server + active_space + * next to the token) is migrated to this layout on first read. */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; @@ -25,35 +36,34 @@ import { parse, stringify } from "yaml"; import { keychainDelete, keychainGet, keychainSet } from "./keychain.ts"; // ============================================================================= -// Constants +// Constants & types // ============================================================================= export const DEFAULT_SERVER = "https://api.memory.build"; -// ============================================================================= -// Types -// ============================================================================= +/** Per-server non-secret config. */ +export interface ServerConfig { + /** Active space slug (the X-Me-Space). */ + active_space?: string; +} -/** - * Per-server credential entry. - */ -export interface ServerCredentials { +/** config.yaml structure. */ +export interface ConfigFile { + default_server: string; + servers: Record; +} + +/** Per-server secrets — the keychain-free fallback. */ +export interface ServerSecrets { session_token?: string; - /** Active space slug (sent as X-Me-Space). */ - active_space?: string; } -/** - * Full credentials file structure. - */ +/** credentials.yaml structure (secrets only). */ export interface CredentialsFile { - default_server: string; - servers: Record; + servers: Record; } -/** - * Resolved credentials for a specific server. - */ +/** Resolved credentials for a specific server. */ export interface ResolvedCredentials { server: string; sessionToken?: string; @@ -67,19 +77,17 @@ export interface ResolvedCredentials { // Path Helpers // ============================================================================= -/** - * Get the config directory path. - * Respects $XDG_CONFIG_HOME, defaults to ~/.config/me. - */ +/** Config directory — respects $XDG_CONFIG_HOME, defaults to ~/.config/me. */ function getConfigDir(): string { const xdg = process.env.XDG_CONFIG_HOME; const base = xdg || join(homedir(), ".config"); return join(base, "me"); } -/** - * Get the credentials file path. - */ +function getConfigPath(): string { + return join(getConfigDir(), "config.yaml"); +} + function getCredentialsPath(): string { return join(getConfigDir(), "credentials.yaml"); } @@ -99,17 +107,14 @@ export function normalizeOrigin(url: string): string { } try { const parsed = new URL(url); - // Remove default ports if ( (parsed.protocol === "https:" && parsed.port === "443") || (parsed.protocol === "http:" && parsed.port === "80") ) { parsed.port = ""; } - // Return origin (scheme + host + port, no trailing slash) return parsed.origin; } catch { - // If URL parsing fails, return as-is with trailing slash stripped return url.replace(/\/+$/, ""); } } @@ -118,124 +123,198 @@ export function normalizeOrigin(url: string): string { // Read / Write // ============================================================================= -/** - * Read the credentials file. Returns empty structure if file doesn't exist. - */ -export function readCredentials(): CredentialsFile { - const path = getCredentialsPath(); - if (!existsSync(path)) { - return { - default_server: DEFAULT_SERVER, - servers: {}, - }; - } +function ensureDir(): void { + const dir = getConfigDir(); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); +} +/** Read config.yaml (non-secret). Empty structure if absent / unparseable. */ +function readConfig(): ConfigFile { + migrateLegacyIfNeeded(); + const path = getConfigPath(); + if (!existsSync(path)) return { default_server: DEFAULT_SERVER, servers: {} }; try { - const content = readFileSync(path, "utf-8"); - const data = parse(content) as Partial | null; + const data = parse( + readFileSync(path, "utf-8"), + ) as Partial | null; return { default_server: data?.default_server ?? DEFAULT_SERVER, servers: data?.servers ?? {}, }; } catch { - return { - default_server: DEFAULT_SERVER, - servers: {}, - }; + return { default_server: DEFAULT_SERVER, servers: {} }; } } -/** - * Write the credentials file atomically with secure permissions. - * Creates the config directory if it doesn't exist. - */ -export function writeCredentials(creds: CredentialsFile): void { - const dir = getConfigDir(); - const path = getCredentialsPath(); +/** Write config.yaml. Non-secret, but the dir is 0700 (owner-only). */ +function writeConfig(config: ConfigFile): void { + ensureDir(); + writeFileSync(getConfigPath(), stringify(config, { lineWidth: 0 })); +} - // Create config directory with 0700 (owner-only) - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true, mode: 0o700 }); +/** Read credentials.yaml (secrets). Empty structure if absent / unparseable. */ +function readSecrets(): CredentialsFile { + migrateLegacyIfNeeded(); + const path = getCredentialsPath(); + if (!existsSync(path)) return { servers: {} }; + try { + const data = parse( + readFileSync(path, "utf-8"), + ) as Partial | null; + return { servers: data?.servers ?? {} }; + } catch { + return { servers: {} }; } +} - const content = stringify(creds, { lineWidth: 0 }); +/** Write credentials.yaml with 0600 (owner read/write only). */ +function writeSecrets(secrets: CredentialsFile): void { + ensureDir(); + writeFileSync(getCredentialsPath(), stringify(secrets, { lineWidth: 0 }), { + mode: 0o600, + }); +} + +/** + * One-time split of a pre-split credentials.yaml — which used to hold + * default_server + per-server active_space alongside the token — into config.yaml + * (non-secret) + a secret-only credentials.yaml. A no-op once config.yaml exists, + * and when there's nothing legacy to move. + */ +function migrateLegacyIfNeeded(): void { + if (existsSync(getConfigPath()) || !existsSync(getCredentialsPath())) return; + + let legacy: { + default_server?: unknown; + servers?: Record< + string, + { session_token?: unknown; active_space?: unknown } + >; + } | null; + try { + legacy = parse(readFileSync(getCredentialsPath(), "utf-8")); + } catch { + return; + } + if (!legacy || typeof legacy !== "object") return; + + const config: ConfigFile = { + default_server: + typeof legacy.default_server === "string" + ? legacy.default_server + : DEFAULT_SERVER, + servers: {}, + }; + const secrets: CredentialsFile = { servers: {} }; + let sawLegacy = typeof legacy.default_server === "string"; + for (const [origin, entry] of Object.entries(legacy.servers ?? {})) { + if (typeof entry?.active_space === "string") { + config.servers[origin] = { active_space: entry.active_space }; + sawLegacy = true; + } + if (typeof entry?.session_token === "string") { + secrets.servers[origin] = { session_token: entry.session_token }; + } + } + if (!sawLegacy) return; // already secret-only — nothing to migrate - // Write with 0600 (owner read/write only) - writeFileSync(path, content, { mode: 0o600 }); + writeConfig(config); + writeSecrets(secrets); } // ============================================================================= -// Server Credential Operations +// Per-server accessors // ============================================================================= -/** - * Get credentials for a specific server. - */ -export function getServerCredentials(server: string): ServerCredentials { - const creds = readCredentials(); - const origin = normalizeOrigin(server); - return creds.servers[origin] ?? {}; +/** Non-secret config for a server (active space). */ +export function getServerConfig(server: string): ServerConfig { + return readConfig().servers[normalizeOrigin(server)] ?? {}; } +/** Secrets for a server (the keychain-free session-token fallback). */ +export function getServerSecrets(server: string): ServerSecrets { + return readSecrets().servers[normalizeOrigin(server)] ?? {}; +} + +// ============================================================================= +// Session token +// ============================================================================= + /** - * Store a session token for a server, and set it as the default. Prefers the OS - * keychain; only when that's unavailable does the token land in the (0600) file. - * Either way the file records the server (default_server) so the token can be - * found again — and any stale file copy is dropped once the keychain has it. + * Store a session token for a server, and record it as the default server. + * Prefers the OS keychain; only when that's unavailable does the token land in + * the 0600 credentials file (and any stale file copy is dropped once the + * keychain has it). The default server is non-secret config (config.yaml). */ export function storeSessionToken(server: string, token: string): void { - const creds = readCredentials(); const origin = normalizeOrigin(server); - if (!creds.servers[origin]) { - creds.servers[origin] = {}; - } + const secrets = readSecrets(); if (keychainSet(origin, token)) { - // Keychain is the source of truth now — don't also persist it in the file. - delete creds.servers[origin].session_token; + if (secrets.servers[origin]) { + delete secrets.servers[origin]; // keychain is the source of truth + writeSecrets(secrets); + } } else { - creds.servers[origin].session_token = token; + secrets.servers[origin] = { session_token: token }; + writeSecrets(secrets); } - creds.default_server = origin; - writeCredentials(creds); + const config = readConfig(); + config.default_server = origin; + writeConfig(config); } -// ============================================================================= -// Space Operations (new model) -// ============================================================================= - /** - * Set the active space (the X-Me-Space) for a server. + * Clear a server's session token from both the keychain and the file. Keeps + * non-secret config (active space, default server). No-op for a token that came + * from $ME_SESSION_TOKEN (we can't unset an env var the user controls). */ -export function setActiveSpace(server: string, spaceSlug: string): void { - const creds = readCredentials(); +export function clearSessionToken(server: string): void { const origin = normalizeOrigin(server); + keychainDelete(origin); - if (!creds.servers[origin]) { - creds.servers[origin] = {}; + const secrets = readSecrets(); + if (secrets.servers[origin]) { + delete secrets.servers[origin]; + writeSecrets(secrets); } - creds.servers[origin].active_space = spaceSlug; - - writeCredentials(creds); } /** - * Clear the active space for a server (e.g. after deleting it). No-op if none - * is set. + * Log out of a server: clear its session secret (keychain + file) but keep the + * non-secret config (active space, default server) so a re-login resumes where + * you left off. */ +export function clearServerCredentials(server: string): void { + clearSessionToken(server); +} + +// ============================================================================= +// Active space (config) +// ============================================================================= + +/** Set the active space (the X-Me-Space) for a server. */ +export function setActiveSpace(server: string, spaceSlug: string): void { + const config = readConfig(); + const origin = normalizeOrigin(server); + if (!config.servers[origin]) config.servers[origin] = {}; + config.servers[origin].active_space = spaceSlug; + writeConfig(config); +} + +/** Clear the active space for a server (e.g. after deleting it). No-op if unset. */ export function clearActiveSpace(server: string): void { - const creds = readCredentials(); + const config = readConfig(); const origin = normalizeOrigin(server); - const entry = creds.servers[origin]; + const entry = config.servers[origin]; if (!entry?.active_space) return; delete entry.active_space; - writeCredentials(creds); + writeConfig(config); } /** * Resolve the active space slug for a server. - * * Priority: --space flag > ME_SPACE env > stored active_space. */ export function resolveSpace( @@ -244,45 +323,7 @@ export function resolveSpace( ): string | undefined { if (flagValue) return flagValue; if (process.env.ME_SPACE) return process.env.ME_SPACE; - return getServerCredentials(server).active_space; -} - -/** - * Clear just the session token for a server, leaving any stored engines and - * API keys in place. Used after the server tells us the session is expired so - * the next command surfaces "Not logged in" instead of repeating the 401. - * - * Clears both the keychain entry and any file copy. No-op for a token that came - * from $ME_SESSION_TOKEN (we can't unset an env var the user controls). - */ -export function clearSessionToken(server: string): void { - const origin = normalizeOrigin(server); - keychainDelete(origin); - - const creds = readCredentials(); - const entry = creds.servers[origin]; - if (entry?.session_token) { - delete entry.session_token; - writeCredentials(creds); - } -} - -/** - * Clear all credentials for a server (keychain token + file entry). - */ -export function clearServerCredentials(server: string): void { - const creds = readCredentials(); - const origin = normalizeOrigin(server); - - keychainDelete(origin); - delete creds.servers[origin]; - - // If we just cleared the default server, reset to default - if (creds.default_server === origin) { - creds.default_server = DEFAULT_SERVER; - } - - writeCredentials(creds); + return getServerConfig(server).active_space; } // ============================================================================= @@ -291,38 +332,36 @@ export function clearServerCredentials(server: string): void { /** * Resolve the active server URL. - * - * Priority: --server flag > ME_SERVER env > default_server in creds > DEFAULT_SERVER + * Priority: --server flag > ME_SERVER env > default_server (config) > DEFAULT_SERVER */ export function resolveServer(flagValue?: string): string { if (flagValue) return normalizeOrigin(flagValue); if (process.env.ME_SERVER) return normalizeOrigin(process.env.ME_SERVER); - - const creds = readCredentials(); - return creds.default_server; + return readConfig().default_server; } /** - * Resolve all credentials for the active server. - * - * The session token (ME_SESSION_TOKEN env > stored) authenticates humans; the - * active space (ME_SPACE env > stored active_space) is the X-Me-Space. An agent - * api key is never persisted — it only ever comes from ME_API_KEY. + * Resolve all credentials for the active server. The session token + * (ME_SESSION_TOKEN env > file > keychain) authenticates humans; the active + * space (ME_SPACE env > config) is the X-Me-Space. An agent api key is never + * persisted — it only ever comes from ME_API_KEY. */ export function resolveCredentials(serverFlag?: string): ResolvedCredentials { const server = resolveServer(serverFlag); - const stored = getServerCredentials(server); + const origin = normalizeOrigin(server); + const config = getServerConfig(server); + const secrets = getServerSecrets(server); return { server, - // env wins; then the file (the keychain-free fallback); then the keychain. - // The token lives in exactly one of file/keychain, so checking the file - // first avoids a keychain lookup on hosts that use the file fallback. + // env wins; then the file (keychain-free fallback); then the keychain. The + // token lives in exactly one of file/keychain, so checking the file first + // avoids a keychain lookup on hosts that use the file fallback. sessionToken: process.env.ME_SESSION_TOKEN ?? - stored.session_token ?? - keychainGet(normalizeOrigin(server)), + secrets.session_token ?? + keychainGet(origin), apiKey: process.env.ME_API_KEY, - activeSpace: process.env.ME_SPACE ?? stored.active_space, + activeSpace: process.env.ME_SPACE ?? config.active_space, }; } From b7b16fe143588864310fb751395f976b1a27ec2b Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 15:33:37 +0200 Subject: [PATCH 091/156] docs: move "api keys for users" from TODO to DECISIONS_FOR_REVIEW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's a pending design/security decision (allow user-minted api keys vs humans-session-only), not build work — so it belongs with the other decisions awaiting a maintainer's sign-off. Co-Authored-By: Claude Opus 4.8 --- DECISIONS_FOR_REVIEW.md | 25 +++++++++++++++++++++++++ TODO.md | 13 ------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index b23988e..6fa9a05 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -96,3 +96,28 @@ call on owner visibility into agent data. The gate is `and p.kind = 'u'` in `packages/database/core/migrate/idempotent/006_membership.sql`. **Status:** needs review. + +--- + +## Should users be able to mint their own API keys? (currently agent-only) + +**Date:** 2026-06-05 · **Area:** auth / api keys + +API keys are currently **agent-only**: `apiKey.create` is gated by +`requireOwnedAgent`, and humans authenticate via session. But the intended CLI +surface treats `ME_API_KEY` as pointing to a "user | agent" and `me apikey +create` as defaulting to self — which implies users can mint their own keys. + +**The decision:** allow user-owned api keys, or keep "humans use sessions only"? + +**Cost if yes (small):** `validate_api_key` already returns the principal +regardless of kind and `authenticateSpace` works unchanged, so it's mostly +relaxing the `apiKey.create` gate to allow `member == self` (a user) in addition +to agents the caller owns. + +**Why it's a real decision:** weigh CLI ergonomics (a user scripting against their +own space without a browser session) against the security stance that human auth +stays interactive/session-only — an api key is a long-lived bearer secret, so +making them mintable for users widens that surface. + +**Status:** needs decision. diff --git a/TODO.md b/TODO.md index 907de5f..349f844 100644 --- a/TODO.md +++ b/TODO.md @@ -3,19 +3,6 @@ Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). -## Reconsider: api keys for users (not just agents) - -Keys are currently agent-only (`apiKey.create` is gated by `requireOwnedAgent`; -humans authenticate via session). The intended CLI surface treats `ME_API_KEY` -as pointing to a "user|agent" and `me apikey create` defaulting to self, which -implies users can mint their own keys. - -- [ ] Decide whether to allow user-owned api keys. `validate_api_key` already - returns the principal regardless of kind, and `authenticateSpace` would - work unchanged — so it's mostly relaxing the `apiKey.create` gate to allow - `member == self` (a user) in addition to agents the caller owns. Weigh - against the "humans use sessions only" security stance. - ## CLI: `me apikey create ` when the agent isn't in the space `apiKey.create` requires the agent already be a member of the active space From 3f3445d7e542082d77580d6efc0924c365f68a15 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Fri, 5 Jun 2026 17:23:47 +0200 Subject: [PATCH 092/156] docs: add REDESIGN.md and a differences report vs. the implementation REDESIGN_DIFFERENCES.md compares the redesign doc against the code on the multiplayer branch: auth in a separate `auth` schema (not core), space-scoped api keys, shipped home/share tree conventions, hardcoded embedding config, the build_tree_access naming, two RPC endpoints, and the unbuilt last-admin safeguard / memory copy / user group list. Co-Authored-By: Claude Opus 4.8 (1M context) --- REDESIGN.md | 848 ++++++++++++++++++++++++++++++++++++++++ REDESIGN_DIFFERENCES.md | 304 ++++++++++++++ 2 files changed, 1152 insertions(+) create mode 100644 REDESIGN.md create mode 100644 REDESIGN_DIFFERENCES.md diff --git a/REDESIGN.md b/REDESIGN.md new file mode 100644 index 0000000..7a592e4 --- /dev/null +++ b/REDESIGN.md @@ -0,0 +1,848 @@ +# Memory Engine (Re)Design + +## Migration Strategy + +Build the new implementation in parallel. When we're happy with it, stand up the database, stop the old server, port/copy/migrate the data from the old prod to the new database, and start the new server. We can practice this migration strategy if needed. This approach also lets us switch embedding models as we migrate; don’t move the embeddings and reembed once moved. + +## V1 Scope and Non-Goals + +This section captures what the first shippable version of the redesign covers and what it deliberately does not. + +### Out of Scope for V1 + +The following are intentionally deferred. They may be revisited in a later version, but they are not built in v1 and code/design should not assume them: + +- Standalone non-OAuth users. User principals are created through OAuth login. Shared service accounts, integrations, and non-human first-class accounts that authenticate only via API keys are valuable but deferred. +- Hosted MCP. The hosted API is JSON-RPC over HTTPS. MCP support exists only in the local stdio proxy. +- Magic/private-path authorization semantics. Private areas, if any, are modeled with explicit tree layout and grants, not with reserved path patterns or implicit deny rules. +- WebSocket and other streaming transports. Bulk import/export uses HTTPS in v1 and may grow chunked endpoints or signed object storage later if needed. +- Actual sharding. The schema and authorization boundary are designed to allow it, but v1 runs all spaces in a single database. +- Billing. The `core` schema may eventually host it, but v1 does not implement billing tables or flows. + +### Non-Goals + +The following are intentionally rejected. Do not build them, and stop and write down the requirement instead if a use case appears to need them: + +- Deny rules and negative access. The access model is monotonic. +- Nested groups. A group may contain users and agents only. +- Agent space-admin or group-admin authority. Agents cannot hold or inherit administrative authority, even via membership in an admin-flagged group. + +## Core + +The `core` schema is a singular, global set of tables. The core manages authentication and authorization. Eventually, it will also handle billing. + +Core SQL does not need to be templated because there is only one core schema. The schema name and table names are stable, and all references should still be schema-qualified for safety. + +### `core.version` + +`core.version` records the current schema version of the global core schema. It is a singleton table: the core schema has exactly one version row. + +The version row lets the server determine whether the core schema is current, needs migration, or is newer than the running server can safely handle. Unlike space schemas, the core schema is singular and global, so core migration state is not scoped by space. + +The version table is intentionally separate from the migration table. The migration table records which steps were applied, but the version row gives the server a cheap compatibility check before doing any real work. If an old server connects to a newer database, it can reject the connection immediately. If the server version matches the database version, it can skip migration checks altogether. + +### `core.migration` + +`core.migration` records which incremental migrations have been applied to the core schema. Each applied migration is recorded once with the target version and timestamp at which it was applied. + +Core migrations use the same incremental/idempotent approach as space schemas. Incremental migrations create or transform durable tables and data exactly once. Idempotent SQL can be re-run safely to refresh functions, triggers, views, policies, and other replaceable database objects. + +Keeping core migration history in `core.migration` makes bootstrap and upgrade behavior explicit and idempotent while keeping it separate from each space's own migration history. + +### `core.space` + +`core.space` enumerates the memory containers in the system. A space is an isolated repository of memories with its own groups and tree access rules. + +Spaces are the user-facing boundary between contexts. A person may have a personal space, belong to an employer's space, and participate in other shared spaces without those memories or access rules accidentally mixing. Memory-oriented commands run in the context of one selected space. Spaces should feel "air gapped." + +Each space has a stable slug used to identify it in URLs, CLI configuration, and the physical schema that stores its large operational tables. The space record also tracks placement information, such as the shard where the space's memory tables live, so the global core schema can route operations to the correct database location. + +### `core.principal` + +`core.principal` stores every identity-like thing that can receive privileges, authenticate, appear in audit fields, or participate in group membership. + +Principals have three kinds: + +- `user`: a first-class principal that is not owned by another principal. This is usually a human OAuth user, but may also be a standalone non-human account such as a shared service account, app, or integration. +- `group`: a collection or capability principal. Groups receive privileges, and users/agents inherit those privileges through `core.group_member`. +- `agent`: a user-owned non-human principal, such as an agent, script, local app, bot, or scheduled job. Agents are used when a human wants a tool to act with attributable, usually narrower, access. + +Agents exist to make agent/script access self-service. A user can create ~~zero or more~~ agents without being a space admin. The owning user can manage the agent's lifecycle and can grant it access up to the access the owner already has. Agents are normal principals for authorization purposes: they can receive direct tree access, belong to groups, authenticate with API keys, and appear in audit fields. + +Principal names follow the scope where people naturally expect them to be unique. User names are global because users are global identities that can participate in many spaces. Group names are space-specific because groups like `engineering`, `admin`, or `design` should be meaningful inside a space without conflicting with similarly named groups in other spaces. Agent names are scoped under their owning user, so multiple users can each have an agent named `claude`, `opencode`, or `importer` without conflict. + +Groups are the only principals that are intrinsically scoped to a single space. A group principal records the space where it is defined, which allows group names to be unique per space. Users and agents are global principals; their relationship to spaces is represented separately through `core.principal_space`. + +With the exception of the initial bootstrap user, principals start with a blank slate: no group membership, no tree access, and no space administrative privileges. Access must be granted directly, inherited through group membership, configured through agent ownership rules, or assigned by setting the space admin flag. + +Agents do not start out with access equal to the owning user. If the access was intended to be equal, there's not much benefit to creating an agent (just use the user itself) (other than attribution in the case that we ever build audit logs). The major feature of having agents is the ability to give them more restricted access. Agents start with blank-slate access. + +### `core.principal_space` + +`core.principal_space` records which principals belong to which spaces. A principal may exist globally without being admitted to every space. To operate in a space, a user, group, or agent must have a `principal_space` row for that space. + +This table is the boundary between global identity and space-local authorization. Users are global and may participate in many spaces. Agents belong to their owning user globally, but must still be admitted to a space before receiving access there. Groups are space-specific; a group principal belongs to exactly one space. + +`principal_space` does not grant memory access on its own. It establishes that the principal is known in the space and records space-local state, including whether the principal is a space admin. Actual memory access comes from `core.tree_access`, either granted directly to the principal or inherited through `core.group_member`. + +Groups still have `principal_space` rows even though their owning space is also recorded on the principal. This deliberate duplication keeps one rule for authorization: any principal that participates in a space has a `core.principal_space` row. It also gives groups the same space-local state as users and agents, including active/disabled state and the space admin flag. + +The `admin` flag on `core.principal_space` is the space-wide administration capability. A principal with `admin = true` can administer users, agents, groups, group membership, invitations, and tree access in that space. If a group has `admin = true`, members of that group inherit space admin authority through `core.group_member`. The admin flag does not itself grant memory visibility; memory visibility and write authority still come from `core.tree_access`. However, an admin can grant tree access to any principal, including themselves. + +The initial user for a space receives `admin = true` and explicit `owner` access to the root tree path. This makes initial single-player use straightforward while keeping space administration and memory visibility represented separately. The system should prevent removing or demoting the last admin principal in a space. + +### `core.space_invitation` + +`core.space_invitation` stores pending invitations for humans to join a space. An invitation is not a principal and does not grant access by itself. It is a pending offer to admit a user principal into a space. + +Invitations are usually addressed to an email address. The invited human may already have a user principal in the system, or they may be entirely new. The system should not create a new principal merely because an invitation was sent. A principal is created or resolved only when the invited person authenticates and accepts the invitation. + +Accepting an invitation creates a `core.principal_space` row for the accepting user in the invited space. If the invitation includes initial group membership, accepting also creates the corresponding `core.group_member` rows. Actual memory access still comes from group membership and `core.tree_access`; the invitation itself never grants direct memory access. + +If invitations are email-based, acceptance should require the authenticated OAuth identity to have a verified email address matching the invitation. Possession of an invite link alone should not be sufficient, because links can be forwarded. + +Invitations should have a lifecycle: pending, accepted, revoked, or expired. They should record who created them, when they were created, when they expire, who accepted them, and who revoked them if revoked. Revocation and expiration prevent future acceptance but do not affect already accepted memberships. + +Creating an invitation requires space admin authority. If the invitation includes initial group membership, the inviter must also be allowed to administer those groups, either through the membership admin flag or through space admin authority. + +Invitations avoid unwanted forced membership. A user is not added to a space until they accept, so unwanted invitations can be ignored, declined, revoked, or allowed to expire. + +### `core.group_member` + +`core.group_member` assigns users and agents to groups within a specific space. Every row is scoped by `space_id`, a group principal, and a member principal. A membership row means the member inherits privileges granted to the group in that space, including tree access and space admin authority. + +Groups are intentionally not nestable. A group may contain users and agents, but it may not contain another group. This avoids recursive membership graphs and keeps the authorization model easier to explain: a user or agent either belongs to a group directly or does not. + +Each membership can carry an `admin` flag. A member with `admin = true` can add and remove members for that group in that space, and can decide whether new memberships also receive the group membership admin flag. + +The `admin` flag on a group membership controls administration of that group only. It lets the member add and remove members for the group, but it does not imply ownership of memories, visibility into all memories, or space-wide administrative authority. + +Groups are the natural way to delegate access for teams. For example, a space admin can create a `project-x` group, add all project team members to that group, and grant the group `owner` access on `projects.x`. Members of `project-x` then inherit ownership of that tree branch and can manage access below `projects.x` without becoming space admins. + +### `core.tree_access` + +`core.tree_access` grants memory access to principals within a specific space. Every row is scoped by `space_id`, a principal, a tree path, and an access level. + +Access applies to the named tree path and all descendants. Granting access on `projects.x` also grants access to `projects.x.design`, `projects.x.budget`, and future children under `projects.x` in the same space. + +Access is a simple ladder: + +- `read`: can search, list, and get memories. +- `write`: includes `read`; can create, update, delete, move, and copy memories. +- `owner`: includes `write`; can grant and revoke access below the owned path. + +The model is monotonic. Grants add access; there is no deny table and no negative access rule. Removing a grant removes that exact grant, but does not create an exception below a broader inherited grant. + +There is no concept of "revoking" tree access. The only mutation primitives are "add grant" and "remove grant" (also called "delete grant"). Both operate on a specific `(space_id, principal_id, tree_path, access)` row. To remove a grant, the caller must specify the grant row that exists. If the requested grant row does not exist, the operation reports an error rather than silently succeeding. + +This is intentional. "Revoke access to `projects.x`" is ambiguous: does it mean "delete the matching grant row," "delete any grant that would imply access to `projects.x` (including ancestor grants)," or "make sure the principal can no longer access `projects.x` by some means"? Forcing the caller to name the exact grant row keeps the semantics explicit. If the caller's expected grant does not exist, the error surfaces the mismatch instead of hiding it behind a no-op. To reduce a principal's effective access on a subtree, remove the specific grant rows that produce that access; if access is inherited from a group, edit the group's grants or the principal's group membership instead. + +Tree access can be granted directly to users, agents, or groups. Group grants are inherited by users and agents through `core.group_member`. Agents receive normal tree access like any other principal, but their owner can self-service grants up to the owner's own access. + +Space admins can administer tree access anywhere in a space, but admin status does not itself imply memory visibility. Visibility and write authority come from `core.tree_access` rows. + +The `owner` access level is the scoped administration mechanism for tree paths. A principal with `owner` access on `private.mat` can grant and revoke access below `private.mat` without involving a space admin. This is the intended mechanism for private user areas: if Mat owns `private.mat`, Mat can decide which users, agents, or groups can access that subtree. + +The same pattern applies to collaborative project areas. Members of a team can be granted `owner` access on `projects.x`, allowing them to manage access for that branch of the tree without becoming space admins. This lets a space delegate administration of specific subtrees while keeping space-wide administration reserved for principals with `core.principal_space.admin = true`. + +### Agent Access + +There is a strong product argument that a user-owned agent should never have more access than its owning user. Agents exist so a user can give a tool attributable and usually narrower access than the user has. If an agent should have fully independent access, it may be better modeled as a first-class `user` principal rather than as a user-owned `agent`. + +There are two possible interpretations of this rule. + +The weaker interpretation is grant-time enforcement. Under this model, the system prevents an owner from granting an agent access the owner does not currently have. This is simple, but it does not preserve the invariant over time. If Alice grants `alice/agent` access to `projects.x` and later loses her own access to `projects.x`, the agent may retain stale access unless the system also finds and revokes it. Furthermore, tree access or group membership that Alice does not have might be granted directly to `alice/agent` by someone other than Alice. + +The stronger interpretation is runtime capping. Under this model, an agent's effective access is always capped by the owner's current effective access. The agent can still have direct tree access and group-derived access, but those grants are masked by the agent owner's grants. The actual access used at runtime is the intersection of the agent's configured access and the owner's current access. + +One implementation option is grant-time enforcement only. It is easy to implement and explain, but it is fragile. Maintaining the invariant would require cascading cleanup whenever an owner loses tree access, is removed from a group, loses space admin status, or otherwise has effective access reduced. + +Another implementation option is runtime access intersection. Conceptually, compute the agent's configured effective access, compute the owner's effective access, and intersect them. For tree access, intersection is tractable because grants are path-prefix rules: overlapping paths produce the more specific path and the lower access level. For example, if the owner has `write` on `projects` and the agent has `read` on `projects.x`, the agent effectively has `read` on `projects.x`. + +Runtime capping is more complex, but it actually enforces the invariant. It also handles later access changes automatically: if the owner loses access, the agent loses effective access without deleting or rewriting the agent's configured grants. + +Space admin status needs special care. The simplest v1 rule is that agents cannot be space admins. This must include inherited space admin authority from groups: if a group has `core.principal_space.admin = true`, an agent's membership in that group should not by itself make the agent an effective space admin. If agents are allowed to carry or inherit the space admin flag, then an agent's effective admin authority should also be capped by the owner, meaning the agent is effectively a space admin only when both the agent and the owner have space admin authority. + +Group membership has the same issue. If agents can belong to groups, group-derived access should still be capped by owner access at runtime. If that proves too complex for v1, a simpler initial version could allow agents to receive only direct tree access masks and defer agent group membership. + +The preferred long-term model is runtime capping: user-owned agents are constrained by the owner's current effective access, while standalone non-human actors that need independent access are modeled as first-class `user` principals. + +**V1** + +- Agents cannot be space admins. +- Agents cannot be group admins. +- Agents may be group members. Group-derived tree access for an agent is intersected with the owning user's current effective access at runtime, so the "agent never exceeds its owner" invariant holds whether access comes from direct grants or group membership. +- Membership in an admin-flagged group does not make an agent an effective admin. The space-admin and group-admin restrictions apply to inherited authority as well. +- Goal: an agent's tree access (direct and group-derived) is capped by the owner's tree access and enforced at runtime. +- Agent group membership should only be removed from v1 if it proves exceedingly difficult to implement correctly. The fallback in that case is agents-with-direct-grants-only, with agent group membership deferred to a later version. + +#### Agent Access Masking Implementation Sketch + +Here is a vibe-coded sketch of what runtime masking might look like. I'm not at all confident in its correctness. It does illustrate that while the masking approach is conceptually simple at face value, it is not straightforward to implement. + +The core masking operation takes two effective access sets: + +- the owner's effective access: `(tree_path, access)` +- the agent's configured effective access: `(tree_path, access)` + +The intersection rule is: + +- Two paths overlap when either path contains the other. +- The effective path is the more specific path. +- The effective access level is the lower of the two access levels. + +In SQL, the masking operation could look like this: + +```sql +with owner_access(tree_path, access) as +( + values + ('projects'::ltree, 1) + , ('projects.x'::ltree, 2) +) +, agent_access(tree_path, access) as +( + values + ('projects.x'::ltree, 1) + , ('projects.y'::ltree, 2) +) +, raw_intersection as +( + select + case + when o.tree_path @> a.tree_path then a.tree_path + when a.tree_path @> o.tree_path then o.tree_path + end as tree_path + , least(o.access, a.access) as access + from owner_access o + inner join agent_access a + on o.tree_path @> a.tree_path + or a.tree_path @> o.tree_path +) +, merged as +( + select + tree_path + , max(access) as access + from raw_intersection + group by tree_path +) +, reduced as +( + select m.* + from merged m + where not exists + ( + select 1 + from merged x + where x.tree_path @> m.tree_path + and x.tree_path <> m.tree_path + and x.access >= m.access + ) +) +select * +from reduced +order by tree_path; +``` + +The `reduced` step removes redundant descendant rows when an ancestor already grants equal or greater access. For example, `projects read` makes `projects.x read` redundant, but `projects.x write` is not redundant because it is stronger than the ancestor grant. + +For future access pushdown, the same operation can consume rendered JSONB access sets: + +```sql +with owner_access as +( + select tree::ltree as tree_path, access::int4 + from jsonb_to_recordset($1) as x(tree text, access int) +) +, agent_access as +( + select tree::ltree as tree_path, access::int4 + from jsonb_to_recordset($2) as x(tree text, access int) +) +-- apply the same intersection, merge, and reduction steps +``` + +Memory operations would use the capped effective access set: + +```sql +where exists +( + select 1 + from effective_access e + where e.access >= 1 + and e.tree_path @> m.tree +) +``` + +Write checks use `e.access >= 2`; scoped administration checks use `e.access >= 3`. + +The important implementation rule is that every authorization check should flow through one effective-access function. User access is direct access plus group-derived access. Agent access is the agent's configured access, including group-derived access, intersected with the owning user's current effective access. + +### Private Areas + +Multiplayer spaces create an immediate product need for private areas. Teams often want a broad shared context that most members can read or write, while still giving each human a place for notes, experiments, drafts, or agent context that should not be visible to everyone else. The desired user experience is something like “give the team write access to everything shared, but not to each person's private area.” + +Two approaches have been proposed. + +One approach is a special carve-out rule: reserve a path pattern such as `private.` or `~` and define root grants to exclude those paths automatically. This would make a broad root grant behave like “everything except private areas.” + +Another approach is to provision spaces with conventional top-level areas, such as `shared` and `private`. Broad team grants would apply to `shared`, while each user would receive owner access to their own subtree under `private`, such as `private.alice` or `private.bob`. + +The motivation is valid: users should not need to design an access model from scratch just to get a normal shared/private collaboration pattern. The open question is whether private areas should be implemented as special authorization semantics or as a recommended tree layout and provisioning convention. + +Magic private paths and implicit deny rules are problematic because they make grants harder to reason about. A grant on root would no longer mean root access; it would mean root access except for paths the system treats specially. That creates surprising behavior for users and makes it harder to explain why a principal can or cannot see a memory. + +They also complicate the access evaluator. The core tree access rule is currently simple: a principal can access a memory when an allowed tree path contains the memory's tree path. Special carve-outs mean every access check must also know about reserved path patterns and subtract them from otherwise valid grants. This pushes the model toward deny semantics even if there is no explicit deny table. + +Deny-like rules become especially awkward with inherited group access. If one group grants broad access and another rule implicitly denies a subtree, the system needs a conflict-resolution policy. Usually denies win, but that means adding a user to a group can unexpectedly remove access, and removing a rule can unexpectedly reveal data. Those interactions are difficult to present clearly in the product. + +Magic paths also constrain future tree organization. If `private.` or another pattern has special meaning, spaces cannot freely use that part of the tree for ordinary memories. The tree becomes partly user-defined and partly reserved by the authorization system, which is exactly the kind of hidden convention the design is trying to avoid. + +Finally, private-path carve-outs make efficient search harder. Memory search needs to combine BM25, HNSW, ltree, metadata, temporal filters, and authorization filtering while continuing to scan until enough authorized results are found. Keeping authorization as a positive set of grant paths maps cleanly to `ltree` containment checks. Subtracting special private paths adds another dimension to every search and makes future access pushdown/sharding more fragile. + +The existing primitives can already model the desired shared/private pattern without special authorization semantics. A space can place shared memories under a known branch such as `shared` or `public`, grant broad team access to that branch, and place per-user private memories under branches such as `private.alice` and `private.bob` with owner access granted only to the corresponding users. + +Under this model, “grant write to everything except private areas” becomes “grant write to `shared`.” The private areas are not exceptions to a root grant; they are simply outside the broadly granted subtree. + +We should defer magic private paths, implicit deny rules, and automatic private area behavior until real usage shows that the explicit tree-layout convention is insufficient. This keeps the v1 access model monotonic, efficient, and explainable. + +### `core.api_key` + +`core.api_key` stores API credentials for non-interactive authentication. A `user` or `agent` principal can have zero or more API keys. Groups cannot have API keys. + +API keys are global credentials for a principal, not credentials for a specific space. After authenticating with an API key, the principal may operate only in spaces where it has been admitted through `core.principal_space` and only with the access granted through `core.tree_access` or inherited through `core.group_member`. + +This keeps key management attached to the principal rather than duplicating credentials per space. A user's agent can use the same key across multiple spaces if it has been admitted to those spaces, while still receiving different access in each space. + +API keys should support independent lifecycle management. Keys can be created, listed, revoked, and rotated without deleting the principal. A user can manage keys for their own agents, and space admins can create/delete keys for any user or agent when as required. + +An API key should be split into a lookup component and a secret component. The lookup component is stored in plaintext for efficient key lookup. The secret component is shown once to the caller and stored only as a strong hash. Authentication succeeds only when both identify the same active key. + +### `core.oauth_identity` + +`core.oauth_identity` stores durable links between OAuth provider identities and user principals. It answers the question: when Google, GitHub, or another OAuth provider says this is user X, which `core.principal` should that authenticate as? + +An OAuth identity belongs to a `user` principal. Agents authenticate with API keys, and groups do not authenticate directly. + +The durable identity key is the OAuth provider plus the provider's stable subject identifier. Email addresses, display names, and avatars are useful profile metadata, but they are not the primary identity because emails can change and may not be verified. + +A user may have multiple OAuth identities linked over time. For example, the same user principal may be linked to both a Google identity and a GitHub identity. A single provider identity should map to only one user principal. + +`core.oauth_identity` should not store transient OAuth state. It is the long-lived account link used after an OAuth flow has completed and the provider identity has been verified. + +### `core.oauth_flow` + +`core.oauth_flow` stores short-lived state for OAuth login flows. This includes the temporary values needed to complete browser-based, CLI, or device-code authentication safely. + +OAuth flows are not durable account links and are not login sessions. They exist only while authentication is in progress. After the flow succeeds, the system links or resolves a `core.oauth_identity` and creates a `core.session`. After the flow fails, expires, or is consumed, the flow record can be removed. + +The flow record should contain enough information to validate the callback or polling request, protect against CSRF/replay, and resume the intended login operation. Depending on the OAuth mode, this may include provider, state, PKCE verifier/challenge data, device code metadata, redirect target, expiration time, and consumption status. + +OAuth flow records should be treated as temporary credentials. They should expire quickly, be single-use where possible, and never grant space access by themselves. + +### `core.session` + +`core.session` stores interactive login sessions created after a user authenticates through OAuth. A user can have one or more active sessions, such as sessions from different machines, browsers, or CLI installations. + +Sessions are global credentials for a user, not credentials for a specific space. After authenticating with a session, the user may operate only in spaces where the user has been admitted through `core.principal_space` and only with access granted through `core.tree_access` or inherited through `core.group_member`. + +Sessions support normal login lifecycle management. They can be created at login, refreshed or extended according to policy, listed for account security, and revoked during logout or credential cleanup. Revoking a session invalidates that session without affecting the user, their other sessions, or their API keys. + +Sessions are for user principals authenticated by OAuth. Agents and standalone non-interactive clients should use `core.api_key` instead. + +## Space + +Each space has a corresponding PostgreSQL schema that holds the space-local operational tables. Space schemas are created on demand when a space is created or first provisioned. + +The DDL for a space schema is rendered from templates. The most important template variable is the schema name itself, because each space has its own schema. All table, index, trigger, and function references in the rendered SQL should be schema-qualified for safety and to avoid accidental dependence on `search_path`. + +Space provisioning also needs per-space configuration. Different spaces may use different embedding models, and different embedding models may have different vector dimensions. The embedding dimension is therefore a template variable used when creating the `embedding` column and vector indexes. The chosen embedding model, embedding dimension, and other space-local database tuning options should be recorded in `core.space` so the server can route embedding work and future migrations correctly. + +This design lets small installations keep all spaces in one database while preserving an operational boundary for future scaling. A space schema can later be placed on a different shard without changing the logical authorization model in `core`. + +## `.memory` + +`.memory` is the primary per-space table. Each row is one memory in the space. This table stays in the space-specific schema because it is the large, search-heavy operational data that will eventually need to scale and shard independently from global authorization metadata (if we are successful). + +Each memory has a UUIDv7 primary key, textual `content`, arbitrary object-shaped JSON metadata in `meta`, a hierarchical `tree` path, optional temporal range information, an optional embedding vector, and timestamps. The `tree` path is the basis for both organization and authorization. The `meta` column supports flexible user-defined structure without creating additional tables for every memory type. + +The table supports three main search dimensions: + +- BM25 full-text search over `content`. +- HNSW vector similarity search over `embedding`. +- Structured filtering over `tree`, `meta`, and `temporal`. + +The `embedding_version` column tracks whether the stored embedding corresponds to the current memory content. When content changes, the embedding is cleared and the version advances so the embedding worker can regenerate the correct vector and ignore stale queue items. + +Temporal values follow one convention. Point-in-time memories use an inclusive single-point range. Period memories use an inclusive-exclusive range. This keeps temporal filtering predictable and avoids ambiguous range boundary behavior. + +## `.embedding_queue` + +`.embedding_queue` is the per-space work queue for embedding generation. Queue rows point to memories in the same space and record the `embedding_version` that should be generated. + +The queue is version-aware. Multiple queue rows may exist for the same memory over time, but workers claim only unresolved rows and can ignore work for older embedding versions when a newer version exists. This prevents stale embedding work from overwriting newer memory content. + +Queue visibility is controlled by `vt`, the visibility time. Workers claim rows whose `vt` is due and whose `outcome` is still null. Attempts and `last_error` record retry history. Finished rows are marked with an outcome such as `completed`, `failed`, or `cancelled` and can later be pruned. + +The queue is space-local for the same reason as `memory`: embedding work is tightly coupled to space-local memory rows and should scale with the memory shard. + +## `.version` + +`.version` records the current schema version of the space-local database objects. It is a singleton table: each space schema has exactly one version row. + +The version row lets the server determine whether the space schema is current, needs migration, or is newer than the running server can safely handle. This check is space-local because spaces may live on different shards and may be migrated independently. + +The version table is intentionally separate from the migration table. The migration table records which steps were applied, but the version row gives the server a cheap compatibility check before operating on the space. If an old server connects to a newer space schema, it can reject the operation immediately. If the server version matches the space schema version, it can skip migration checks for that space altogether. + +The table should also record when the version was last updated so migrations and operational tooling can inspect the state of a space without relying only on global metadata. + +## `.migration` + +`.migration` records which incremental migrations have been applied to a space schema. Each applied migration is recorded once with the target version and timestamp at which it was applied. + +The migration table makes space provisioning and upgrades idempotent. When a migration runs, the migrator can skip incremental migrations that have already been recorded and apply only the missing ones. After incremental migrations complete, idempotent SQL can be re-run safely to refresh functions, triggers, and other replaceable database objects. + +Keeping migration history inside the space schema makes each space self-describing. This is useful when spaces are created on demand, upgraded independently, or eventually moved across shards. + +## Authorization Boundary + +Authorization metadata lives in the global `core` schema. This includes spaces, principals, group membership, tree access, OAuth identities, sessions, and API keys. The per-space schemas hold the large operational data: `memory` and `embedding_queue`. + +Keeping authorization metadata in `core` is important because effective access depends on several related facts: principal kind, space membership, space admin state, group membership, group administration, direct tree access, group-derived tree access, agent ownership, and the owner's own effective access. Resolving that graph should happen in one transactional context over tables with real foreign key constraints. + +The boundary between authorization and memory operations should be an effective access set. Memory operations should not know how to interpret principals, groups, agents, or space admin state. They should consume rows shaped like: + +```sql +(tree_path ltree, access int4) +``` + +The core schema should expose a function similar to: + +```sql +core.effective_tree_access +( _space_id uuid +, _principal_id uuid +) +returns table +( tree_path ltree +, access int4 +) +``` + +This function is responsible for resolving direct access, group-derived access, and agent access masking. For a user, effective access is direct tree access plus group-derived tree access. For an agent, effective access is the agent's configured access, including group-derived access, intersected with the owning user's current effective access. + +Initially, space-specific memory functions can call `core.effective_tree_access(...)` directly in a materialized CTE. This preserves referential integrity for principals, groups, and tree access while keeping access evaluation inside SQL where BM25, HNSW, and tree indexes can be used correctly. + +```sql +with effective_access as materialized +( + select * + from core.effective_tree_access($1, $2) +) +select m.* +from space_slug.memory m +where exists +( + select 1 + from effective_access a + where a.access >= 1 + and a.tree_path @> m.tree +); +``` + +In a future sharded implementation, the coordinator can call the same core function, serialize the result, and pass it to the shard-local memory function. + +```sql +select jsonb_agg +( + jsonb_build_object + ( 'tree', tree_path::text + , 'access', access + ) +) +from core.effective_tree_access($space_id, $principal_id); +``` + +A pushed-down access set would be a small list of tree paths and access levels, for example: + +```json +[ + { "tree": "projects.x", "access": 2 }, + { "tree": "shared.docs", "access": 1 } +] +``` + +The shard-local function would parse the JSONB as a recordset, materialize it, and join memories against it. + +```sql +with effective_access as materialized +( + select tree::ltree as tree_path, access::int4 + from jsonb_to_recordset($access_jsonb) + as x(tree text, access int) +) +select m.* +from space_slug.memory m +where exists +( + select 1 + from effective_access a + where a.access >= 1 + and a.tree_path @> m.tree +); +``` + +Clients and agents must never provide this access set directly. It is produced only by trusted server-side code after authentication and authorization. In the sharded version, the rendered access set is a snapshot for a trusted operation. If authorization changes after rendering but before shard execution, the shard executes against the rendered snapshot. + +This gives us the simple first implementation while preserving a migration path for vertical scaling limits and eventual sharding. The memory layer always consumes effective access; only the source of that effective access changes. + +## Deletion and Cascading + +### No Soft Deletes + +One cardinal rule: no soft deletes. Anywhere. Ever. + +Soft deletes (`deleted_at`, `archived_at`, `is_deleted`, `active = false`, or any other "tombstone in the live table" pattern) cause problems that compound over time: + +- They break unique constraints. A `name` column that should be unique now has to be unique-among-non-deleted, which means partial indexes and conditional uniqueness logic everywhere. +- They break foreign key constraints. Downstream rows can keep pointing to "deleted" parents, so every join has to filter on the soft-delete flag or risk leaking removed data. +- They bloat tables with rows nobody is supposed to see. Production tables can end up 90% dead rows, with the database wading through garbage to satisfy every query. +- They make application code ambiguous. Every query has to remember to exclude soft-deleted rows. Forgetting once is a bug; forgetting in a search query is a security bug. + +The rule is simple: tables represent live state. If a row is no longer live, hard delete it. + +### When to Hard Delete + +Prefer hard deletes by default. Specifically: + +- `core.session`: hard delete on logout, revoke, or expiry cleanup. +- `core.oauth_flow`: hard delete after consumed, failed, or expired. +- `core.api_key`: hard delete on revoke. +- `core.space_invitation`: hard delete on revoke, expiry, or after the invitation has been accepted and no longer needs to be visible. +- `core.group_member`: hard delete when membership is removed. +- `core.tree_access`: hard delete when a grant is revoked. +- `core.principal_space`: hard delete when a principal is removed from a space. +- `core.principal` (groups, agents, users): hard delete when removed, subject to cascade rules below. +- `.embedding_queue`: hard delete completed, failed, and cancelled rows via periodic cleanup. +- `.memory`: hard delete on memory delete. + +### Expiry Cleanup + +Some rows are inherently time-bounded: sessions, OAuth flows, invitations, embedding queue outcomes. These should have a periodic cleanup process that hard deletes rows past their expiry or retention window. Expired rows are not soft-deleted to "remember they existed"; they are removed. + +If a particular table needs longer retention for operational debugging, the retention window should be configurable and enforced by the cleanup process, not by leaving expired rows in the live table indefinitely. + +### Audit / Dead Tables + +If we ever do need to keep evidence of a deleted row, the row must move out of the live table into a separate audit or dead table. The live table only holds live state; history goes elsewhere. + +This pattern is opt-in per table. V1 does not require audit tables anywhere. If a future feature needs them (for example, compliance, billing reconciliation, or security forensics), we add a dedicated table such as `core.audit_event` or `core.dead_api_key` and write to it from the same transaction that performs the hard delete. + +### Cascading Deletes + +Forcing administrators to hand-revoke every grant and membership before deleting a principal is bad UX. Commands that delete parent objects should expose cascade behavior for their expected dependents rather than failing with a wall of FK errors. + +The conventions are: + +- `--cascade`: also delete dependent rows that would otherwise block the operation. Refers to expected, documented dependent rows for that command. +- `--force`: skip confirmation prompts. `--force` does not mean "ignore integrity"; it means "I already know what this will do." + +Commands may require `--cascade`, `--force`, or both for destructive operations. The default behavior without flags should be safe and explain what is blocking the delete. + +#### Deleting a Group + +`me group delete [--cascade]` should cascade to: + +- `core.group_member` rows where the group is the group. +- `core.tree_access` rows granted to the group. +- `core.principal_space` row for the group. +- The group's `core.principal` row. + +This matches intent: the group no longer exists, so its memberships and grants no longer exist either. + +#### Deleting an Agent + +`me agent delete [--cascade]` should cascade to: + +- API keys owned by the agent. +- Group memberships for the agent in every space. +- Direct `core.tree_access` grants to the agent in every space. +- `core.principal_space` rows for the agent. +- The agent's `core.principal` row. + +#### Removing a Principal from a Space + +`me space member remove [--cascade]` removes that principal from the named space and cascades to: + +- The principal's group memberships in that space. +- Direct tree access grants to that principal in that space. +- The principal's `core.principal_space` row for that space. + +If the principal is a user with owned agents, the cascade also removes those agents from the same space (their `principal_space` row and any space-local memberships and grants). The global agent principal and its API keys remain, because API keys and agents are global, not space-scoped. + +The user, their global agents, and their API keys are not deleted by this command. To delete the user globally, use a separate command. + +#### Deleting a Space + +`me space delete [--force]` is the most destructive operation. It removes: + +- All `core.principal_space` rows for the space. +- All `core.group_member` rows in the space. +- All `core.tree_access` rows in the space. +- All group principals scoped to the space. +- All `core.space_invitation` rows for the space. +- The space's per-schema operational data: `.memory`, `.embedding_queue`, `.version`, `.migration`, and the schema itself. +- The `core.space` row. + +This command should always require `--force` or an explicit confirmation prompt. + +### Last-Admin Safeguard + +The cascade rules above do not override the invariant that a space must always have at least one admin principal. Any cascade that would remove the last `core.principal_space.admin = true` principal in a space must fail rather than leave the space adminless. The error should name the conflicting principal so the operator can promote a replacement before retrying. + +## The API Server + +JSON-RPC over HTTPS. + +## API, Client, and MCP Boundary + +The hosted API server exposes JSON-RPC over HTTPS. JSON-RPC gives us a simple, stable, application-owned protocol that we can shape around Memory Engine's product and authorization model without inheriting MCP-specific compatibility constraints. + +Request and response schemas live in a shared Zod model package. This package is the source of truth for the JSON-RPC method contract and is shared by the API server and TypeScript client. The API server uses the schemas to validate requests and shape responses. The client package depends on the schema package and provides a thin typed wrapper around calling the hosted JSON-RPC API. + +The CLI and local MCP server both use the client package. This keeps transport and tool-specific concerns out of the API server and avoids duplicating API call logic. + +Both the schema package and client package should be published to npm. This lets external TypeScript consumers call the hosted API directly without going through the CLI or MCP layer. The client package makes it easy to use Memory Engine from TypeScript scripts with full typing and minimal boilerplate. + +JSON-RPC over HTTPS is also much easier to integrate with scripts and services in other languages than a hosted MCP server would be. Any language that can make an HTTPS request and parse JSON can call the hosted API directly. A hosted MCP server, by contrast, would require MCP transport, framing, tool registration semantics, and per-client/per-model compatibility handling, which is heavy for ad-hoc scripts and integrations in languages that do not have a Memory Engine client library. + +HTTPS does have one tradeoff worth noting. The previous version of Memory Engine used JSON-RPC over WebSockets, which allowed streaming for bulk imports, exports, and unusually large memories. JSON-RPC over HTTPS imposes request and response payload size limits, which is more constraining for those operations. We accepted that tradeoff because the operational and client complexity of WebSockets outweighed the streaming benefit for the common case. If bulk and streaming workloads become important, we can revisit by adding chunked endpoints, signed object storage uploads/downloads, or a streaming transport for specific operations rather than reverting the entire API. + +We explicitly are not making the hosted API server an MCP server in the initial design. MCP is valuable for integrating with AI agents, but it brings extra protocol overhead and model/client compatibility constraints. Different MCP clients and model providers handle optional, nullable, tuple, record, and JSON Schema details differently. For example, some clients omit optional fields, while others send explicit `null`; some model/tooling paths render nullable unions poorly or reject certain schema shapes. + +The local MCP server is therefore a stdio proxy. It exposes MCP tools to agents, handles MCP-specific schema compatibility, normalizes model/client quirks, and forwards calls to the hosted JSON-RPC API through the client package. This isolates MCP complexity in one local integration layer while keeping the hosted API clean and flexible. + +The intended dependency flow is: + +``` +shared zod models + -> API server + -> client package + -> CLI + -> local MCP server +``` + +The hosted API remains JSON-RPC. The local MCP server adapts MCP tool calls into JSON-RPC client calls. A hosted MCP server is not ruled out, but it is not part of the current implementation plan. If we add hosted MCP later, it should be an adapter over the same JSON-RPC/client boundary rather than replacing the application API. + +## Environment Vars and Global CLI Options + +- `ME_SERVER` or `--server` \- the URL of the API (dev/prod/self-host) +- `ME_API_KEY` or `--api-key` \- an api key pointing to a specific user|agent +- `ME_SPACE` or `--space` \- the slug of a space to scope commands to + +## Config Files + +We need to store session tokens somewhere. We need to store the currently selected space somewhere. + +Use keychain so long as we keep proper scoping between servers + +## Authentication Commands + +### `me login [space_id|slug|name]` + +Authenticates with the system via OAuth and creates a session token. Session token should be saved in a known file. + +After authentication, `me login` selects a current space using the following rules: + +- If the user belongs to exactly one space, auto-select it. +- If the user belongs to multiple spaces, use the space specified by the positional argument, `--space`, or `ME_SPACE`. +- If the user belongs to multiple spaces and no space was specified, show an interactive picker when stdin/stdout is a TTY. Outside a TTY, exit with an error indicating that `--space` or `ME_SPACE` is required. +- If the user belongs to no space, select none. Subsequent commands that require a space error out with guidance to create a space or accept an invitation. + +The selected space is stored alongside the session credentials and scoped per server, so future invocations resume the same space without re-prompting. + +### `me logout` + +Expires the current session token. Removes it from the file. + +### `me whoami` + +Displays info about the principal and possibly OAuth stuff. + +## Space Commands + +### `me space use ` + +### `me space create` + +Creates a new space. + +V1 does not provision any out-of-the-box tree organization. A newly created space starts with an empty tree. Space admins set up whatever layout they want. For single-player mode, most users will just start writing memories at whatever paths make sense to them. + +The creating user receives: + +- `core.principal_space.admin = true` for the new space. +- `owner` access on the root tree path, so they can grant and revoke access anywhere below it. + +Stretch goal (only if time permits): a `--template ` or `--multiuser` flag that provisions some out-of-the-box structure for collaborative spaces (for example, a `shared` branch with team grants and `private.` branches owned by individual users). This is explicitly optional for v1 and should not block shipping. + +### `me space delete` + +### `me space alter` + +### `me space list` + +### `me space invite` + +### `me space invite list` + +### `me space invite revoke` + +## User Commands + +User principals are created via OAuth login and managed primarily through identity, invitation, and space membership commands. The `me user` command surface is therefore mostly unneeded for the immediate next version. + +Standalone non-OAuth users (for example, shared service accounts, integrations, or non-human first-class accounts that authenticate only via API keys) are valuable in the longer term, but we are deferring them until after the initial release. When we add them, this section will define commands for creating, renaming, deleting, and inspecting standalone user principals. + +For now, the only intentionally supported user-management surface is: + +### `me user group list ` + +Lists the groups the named user belongs to in the current space. + +## Agent Commands + +### `me agent create ` + +Creates an agent principal owned by the current user. + +### `me agent delete ` + +Deletes an agent principal owned by the current user. + +### `me agent rename ` + +Creates an agent principal owned by the current user. + +### `me agent group list ` + +probably more commands here + +## Group Commands + +Group commands must either have ME\_SPACE or \--space specified, or they use the currently `use`d space. + +### `me group create ` + +You must be a space admin to create new groups. + +### `me group delete ` + +You must be a space admin to delete groups. + +### `me group rename ` + +You must be a space admin to alter groups. + +### `me group member add ` + +You must be a space admin or be a member of the group with the `admin` flag on the membership in order to add members. Member must be in the `core.principal_space` table for this space. Member cannot be another group. + +### `me group member remove ` + +You must be a space admin or be a member of the group with the `admin` flag on the membership in order to remove members. Member must be in the `core.principal_space` table for this space. + +### `me group member list ` + +Any user|agent in `core.principal_space` for the space may list members of a group in the space. + +## Access Commands + +There is no `me access revoke`. The only mutation verbs are `grant` and `rm-grant`, matching the `core.tree_access` semantics: a grant is a specific `(principal, tree_path, access)` row, and removing one removes that exact row. + +### `me access grant ` + +Creates a `core.tree_access` row for the principal at the given tree path with the given access level. If an equivalent grant already exists, the command reports that fact rather than silently succeeding. + +### `me access rm-grant ` + +Deletes the specific grant row identified by `(principal, tree_path, access)`. If no such row exists, the command errors out. This is intentional: callers must name the exact grant they intend to remove, so unexpected "missing" grants surface as errors instead of silent no-ops. + +`rm-grant` does not cascade to ancestor or descendant grants and does not affect access inherited through group membership. To reduce inherited access, remove the relevant group's grant or change the principal's group membership. + +### `me access list ` + +Lists all `core.tree_access` rows for the principal in the current space, including both direct grants and group-derived grants (clearly labelled). + +### `me access list ` + +Lists all `core.tree_access` rows in the current space whose path is an ancestor of, equal to, or a descendant of the given path, so admins can see who has access to a given subtree. + +## API Key Commands + +### `me apikey create ` + +If not specified, lists api keys for self. Otherwise, must be an agent owned by the user. + +### `me apikey list ` + +If not specified, lists api keys for self. Otherwise, must be an agent owned by the user. + +### `me apikey revoke ` + +API key must belong to the user or an agent owned by the user. + +## Memory Commands + +Can we make `memory` optional? Can the memory commands be top-level? + +### `me [memory] create` + +### `me [memory] get ` + +### `me [memory] edit ` + +### `me [memory] patch|update` + +### `me [memory] delete|rm ` + +### `me [memory] delete|rm --tree ` + +### `me [memory] tree --tree --levels` + +### `me [memory] move|mv ` + +### `me [memory] copy|cp ` + +### `me [memory] search` + +### `me [memory] import` + +### `me [memory] export` + +## MCP Server + +### Local MCP `me mcp` + +Runs a stdio MCP server locally scoped to a space and user. The MCP server proxies to the hosted API. Uses an API key via either ME\_API\_KEY or \--api-key. + +The benefit of a local MCP server is that it can import/export to/from files without reading the contents through the context window (although, this is something of a security hole). + +- me\_memory\_get +- me\_memory\_search +- me\_memory\_update|patch +- me\_memory\_delete +- me\_memory\_tree +- me\_memory\_import +- me\_memory\_export + +### Hosted MCP + +No file-related tools. More thought needs to go here. + +- me\_memory\_get +- me\_memory\_search +- me\_memory\_update|patch +- me\_memory\_delete +- me\_memory\_tree diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md new file mode 100644 index 0000000..61f909d --- /dev/null +++ b/REDESIGN_DIFFERENCES.md @@ -0,0 +1,304 @@ +# REDESIGN.md vs. Current Implementation — Differences + +This document compares `REDESIGN.md` against the code as it actually exists on +the `multiplayer` branch. It is organized as: + +1. TL;DR of the substantive divergences +2. Architectural divergences (design intent changed) +3. Naming / shape divergences (same idea, different surface) +4. Not yet implemented (gaps vs. the redesign) +5. Implemented beyond the redesign (exists in code, absent from the doc) +6. Confirmed matches (the redesign describes reality) +7. Doc-hygiene notes for REDESIGN.md itself + +File references are `path:line` against the repo root. + +--- + +## 1. TL;DR — the substantive divergences + +| # | Topic | Redesign says | Implementation does | Severity | +|---|-------|---------------|---------------------|----------| +| A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | +| B | API keys | **Global** per-principal; one key works across many spaces | **Space-scoped**: format `me...`, validated against `X-Me-Space` | **Major** | +| C | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | +| D | Embedding config | Per-space model/dimension, recorded in `core.space` | Hardcoded `text-embedding-3-small` / 1536 for all spaces; `core.space` has a TODO comment, no such columns | **Major** | +| E | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | +| F | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | +| G | Last-admin safeguard | Must prevent removing/demoting the last admin | **Not implemented** | Gap | +| H | `me memory copy`/`cp` | Listed | **Not implemented** | Gap | +| I | `me user group list` | The one supported user command for v1 | **Not implemented** | Gap | + +The agent access-masking model (the part the doc was least confident about) **is** +implemented as designed — see §6. + +--- + +## 2. Architectural divergences + +### A. Auth lives in a separate `auth` schema, not in `core` + +The redesign places all authentication state in `core`: `core.session`, +`core.oauth_identity`, `core.oauth_flow`. The implementation instead uses a +dedicated **`auth` schema** shaped like better-auth, and `core` contains no auth +tables at all. + +Actual `auth` schema (`packages/database/auth/migrate/incremental/`): + +- `auth.users` (`001_users.sql:5`) — `id, name, email, email_verified, image, created_at, updated_at`. **`auth.users.id == core.principal.id`** for user principals. +- `auth.accounts` (`002_accounts.sql:8`) — OAuth provider links (`provider_id` ∈ {google, github}, tokens, scope). This is the redesign's `oauth_identity`. +- `auth.sessions` (`003_sessions.sql:10`) — `token_hash` (sha256), `expires_at`, `ip_address`, `user_agent`. This is the redesign's `session`. +- `auth.device_authorization` (`004_device_authorization.sql:7`) — device-code flow state. This is the redesign's `oauth_flow`. +- `auth.verifications` (`005_verifications.sql:7`) — present for better-auth shape parity. + +Net effect: the "authorization boundary lives entirely in `core`" framing in the +redesign (§"Authorization Boundary") is true for *authorization* (principals, +grants, groups) but **not** for *authentication* — authentication is its own +schema. The redesign never mentions an `auth` schema or better-auth. + +### B. API keys are space-scoped, not global + +The redesign is explicit (§`core.api_key`): "API keys are global credentials for +a principal, not credentials for a specific space… A user's agent can use the +same key across multiple spaces." + +The implementation does the opposite. The key format is +`me...` (`packages/engine/core/api-key.ts:4`, +`packages/server/middleware/authenticate-space.ts:9`), so the **space slug is +baked into the key**. `authenticate-space.ts:114` rejects the request if the +key's slug doesn't match the `X-Me-Space` header. A key therefore authenticates +into exactly one space; using an agent across N spaces requires N keys. + +`core.api_key` (`006_api_key.sql:4`) columns: `id, member_id (→ principal.member_id, +u|a only), lookup_id (unique, 16-char), secret (sha256), name, created_at, +expires_at`, with a unique `(member_id, name)`. Note the table itself has **no +`space_id` column** — the scoping is purely via the slug embedded in the +presented key string, not a DB relationship. This is worth reconciling: the +storage is principal-global (as the redesign wants) but the credential string is +space-bound (as the redesign rejects). + +### C. Reserved tree paths and provisioning are built, not deferred + +This is the largest behavioral divergence. The redesign's V1 scope says (§"Private +Areas", §"me space create"): + +- "We should **defer** magic private paths, implicit deny rules, and automatic + private area behavior." +- "V1 does not provision any out-of-the-box tree organization. A newly created + space starts with an empty tree." +- "The creating user receives… `owner` access on the **root** tree path." + +The implementation instead bakes in two reserved roots and provisions them: + +- `home.` with `~` as input sugar — `HOME_NAMESPACE` and `homePrefix()` + in `packages/database/space/path.ts:29,60`. `add_principal_to_space(...)` + (`core/migrate/idempotent/006_membership.sql`) auto-grants a joining **user** + `owner` on `home.`. +- `share` — `SHARE_NAMESPACE` in `packages/database/space/path.ts:38`; a bare + `memory.create` with no `tree` defaults here. +- A space **creator** gets `admin` + `owner@home.` + `owner@share`, and + **explicitly not `owner@root`** — `packages/server/provision.ts:55` (`addSpaceCreator`). + This is the opposite of the redesign's "creator gets owner@root." + +Functionally these are still ordinary positive ltree grants (no deny rules, no +implicit subtraction), so the redesign's *non-goal* of "no negative access" is +respected. But the **convention layer the redesign deferred is shipped**, and the +creator's grant is `home`+`share` rather than `root`. Any reader of REDESIGN.md +would expect a fresh space to be empty and root-owned; it is neither. + +### D. Embedding model/dimension is hardcoded, not per-space + +The redesign (§"Space") wants the embedding model and dimension to be per-space, +templated into the DDL, and recorded in `core.space` so the server can route +embedding work. + +Reality is split: + +- The DDL **is** templated: `embedding halfvec({{embedding_dimensions}})` + (`space/migrate/incremental/001_memory.sql:10`). So the *mechanism* exists. +- But the value is **hardcoded to 1536 / `text-embedding-3-small` for every + space** server-side (`packages/server/config.ts:8`, `packages/server/index.ts:212` + comment: "Model and dimensions are hardcoded - all spaces use the same + embedding model"). +- `core.space` does **not** record provider/model/dimension. It literally carries + a TODO: `-- we likely need columns for embedding provider, model, dimensions` + (`core/migrate/incremental/001_space.sql:9`). There is also **no shard / + placement column**, though the redesign (§`core.space`) says the space record + "tracks placement information, such as the shard." + +So the "per-space embedding" and "placement metadata in `core.space`" parts of the +redesign are not realized; all spaces are uniform and single-DB (consistent with +the "no sharding in v1" non-goal, but the metadata hooks the redesign called for +aren't there yet). + +--- + +## 3. Naming / shape divergences (same concept, different surface) + +### E. Access resolution function + +- Redesign: `core.effective_tree_access(_space_id uuid, _principal_id uuid) + returns table(tree_path ltree, access int4)`. +- Actual: `core.build_tree_access(_member_id uuid, _space_id uuid) returns jsonb` + (`core/migrate/idempotent/003_tree_access.sql:131`). Differences: **name**, + **argument order**, takes **`_member_id`** (not `principal_id`), and returns a + **JSONB array** of `{tree_path, access}` objects rather than a SQL table. + +Space functions consume it as a `_tree_access jsonb` argument +(`space/migrate/idempotent/001_memory.sql:33`), matching the redesign's +"pushed-down JSONB access set" future shape — but that's the *only* code path, +not a later optimization. Per CLAUDE.md the auth gate is "non-empty +`build_tree_access`." + +### F. Two RPC endpoints, plus REST auth + +The redesign describes "the hosted API server exposes JSON-RPC over HTTPS" as a +single surface. The implementation splits it (`packages/server/router.ts:252`): + +- `/api/v1/memory/rpc` — session **or** api-key + required `X-Me-Space`. Hosts + `memory.*`, `principal.*`, `group.*`, `grant.*`, `invite.*`, `apiKey.*`. +- `/api/v1/user/rpc` — **session only** (api keys rejected here). Hosts `whoami`, + `agent.*`, `space.*`. +- `/api/v1/auth/*` — REST device-flow endpoints (`device/code`, `device/token`, + `device/verify`, `device/approve`, `callback/:provider`). + +The split (agents can't manage agents/spaces) is a real design decision absent +from the doc. + +### CLI verb renames (vs. the redesign's command list) + +All same-intent, different spelling: + +- `me space alter` → **`me space rename`** (`commands/space.ts:212`). +- `me agent group list` → **`me agent groups`** (`commands/agent.ts:161`). +- `me apikey revoke` → **`me apikey delete`/`rm`** (+ a `me apikey get`) + (`commands/apikey.ts`). The rename aligns with the doc's own "no soft delete / + hard delete" stance, but the doc text still says `revoke`. +- `me group member add/remove/list` → **`me group add` / `remove`(`rm-member`) / + `members`** (`commands/group.ts`). + +--- + +## 4. Not yet implemented (gaps vs. the redesign) + +- **G. Last-admin safeguard.** The redesign requires that no cascade or removal + may strip the last `principal_space.admin = true` from a space (§"Last-Admin + Safeguard", §`core.principal_space`). No such check exists in SQL or app code. +- **H. `me memory copy` / `cp`.** Listed in §"Memory Commands"; `move`/`mv` + exists, `copy`/`cp` does not (`commands/memory.ts`). The MCP server likewise has + `me_memory_mv` but no copy tool. +- **I. `me user group list `.** The redesign names this the single + supported `me user` command for v1; there is **no `me user` command** at all. +- **Verified-email enforcement on invite acceptance.** The redesign (§`core.space_invitation`) + wants acceptance to require an OAuth-verified email matching the invitation + ("possession of an invite link alone should not be sufficient"). Invitations + are implemented (`invite create/list/revoke`, `redeem_space_invitations`), but + the verified-email match requirement should be confirmed against + `009_invitation.sql` / redemption before relying on it. + +--- + +## 5. Implemented beyond the redesign (in code, not in the doc) + +- **`me agent add `** — adds one of your global agents to the active space + (`commands/agent.ts:134`). The redesign treats agent→space admission as implied + by `principal_space` but never gives it a command. (Agents are global; they must + be admitted to a space before they can hold a key/grants there.) +- **Client/version gating.** `X-Client-Version` header check returns HTTP 426 + "Upgrade Required" below `MIN_CLIENT_VERSION` (`server/middleware/client-version.ts`), + and the migrator rejects an app older than the DB version + (`database/migrate/kit.ts:334`). The redesign mentions version *tables* for + compatibility but not a client-version handshake. +- **`core.space.language`** column for the BM25 text-search config + (`001_space.sql:8`) — per-space text language, not mentioned in the redesign. +- **Extra env vars:** `ME_SESSION_TOKEN` and `ME_NO_KEYCHAIN` (`packages/cli/credentials.ts`, + `packages/cli/keychain.ts`) beyond the doc's `ME_SERVER`/`ME_API_KEY`/`ME_SPACE`. +- **CLI config split + keychain.** Implemented as `~/.config/me/config.yaml` + (non-secret, per-server `active_space`) + `credentials.yaml` (0600 secret + fallback) + OS keychain (macOS `security`, Linux `secret-tool`). The redesign + only says "store session tokens somewhere… use keychain"; the concrete split is + an elaboration. +- **Extra CLI surface:** `me serve` (web UI), `me pack`, `me claude` / `me codex` + / `me gemini` / `me opencode` (install + import), `me completions`, `me version`, + `me upgrade`. The doc only asks about a few of these. +- **MCP tool set is broader than the doc's list.** Actual tools + (`packages/cli/mcp/server.ts`): `me_memory_create`, `me_memory_get`, + `me_memory_search`, `me_memory_update`, `me_memory_delete`, + `me_memory_delete_tree`, `me_memory_mv`, `me_memory_tree`, `me_memory_import`, + `me_memory_export`. The redesign's local list omits `create`, `mv`, and + `delete_tree`, and writes `update|patch` (there is a single `update`, no + `patch`). +- **Transitive group admin (Model 2)** is implemented (`is_principal_space_admin`, + `member_groups` in `003_tree_access.sql` / `001_principal_space.sql`): a user in + an admin-flagged group inherits space admin; agents are excluded. The redesign + discusses this but the concrete enforcement is in code. + +--- + +## 6. Confirmed matches (the redesign describes reality) + +These are worth recording because they are the parts the redesign was least sure +about or most opinionated on, and the code honors them: + +- **Agent access runtime capping.** The "stronger interpretation" the redesign + preferred — an agent's effective access is its configured access (direct + + group-derived) **intersected with the owner's current effective access at + runtime** — is implemented in `agent_tree_access` within + `core/migrate/idempotent/003_tree_access.sql:54` (overlap → more-specific path, + `least(access)`, then reduce redundant descendants). The doc's "vibe-coded + sketch" became real SQL. The V1 rules hold: agents can't be space admins + (`is_principal_space_admin` excludes `kind='a'`), can't be group admins + (`member_groups` zeroes the admin flag for agents), may be group members, and + inherited admin from an admin-flagged group does not make an agent an admin. +- **No soft deletes.** No `deleted_at`/`archived_at`/`is_deleted`/`active` + anywhere; hard deletes via FK `on delete cascade` plus explicit cascade + functions (`remove_principal_from_space`, etc.). Matches §"Deletion and + Cascading". +- **Principal model.** `kind ∈ {u,a,g}`, generated `member_id` (u|a), generated + `user_id`/`agent_id`/`group_id`, agent `owner_id → principal(user_id)`, name + scoping (users global, agents per-owner, groups per-space). Matches §`core.principal`. +- **`tree_access` ladder** read=1 / write=2 / owner=3, applies to path + all + descendants, monotonic, no deny table. Matches §`core.tree_access`. CLI verbs + `grant` / `rm-grant` (no `revoke`) match the doc exactly (`commands/access.ts`). +- **Space schema tables** `.memory`, `.embedding_queue`, + `.version`, `.migration`; `embedding_version` on memory; version-aware + queue with `vt` / `outcome` / `attempts` / `last_error`; temporal `[a,a]` vs + `[start,end)` convention enforced by check constraint. Matches §"Space" / §`.*`. +- **Transport & boundaries.** JSON-RPC over HTTP (not WebSockets), no hosted MCP + server, local **stdio** MCP proxy that forwards through the client package using + `ME_API_KEY`; `@memory.build/protocol` (Zod) is the contract source of truth and + both `protocol` and `client` are published (`private: false`). Dependency flow + protocol → server → client → CLI → MCP holds. Matches §"API, Client, and MCP + Boundary". +- **`me memory` is optional / top-level.** Both `me memory ` and `me ` + (e.g. `me search`, `me create`) work — answers the doc's open question "Can the + memory commands be top-level?" with "yes." +- **Auth specifics** the doc implied: GitHub/Google OAuth, device-code flow, + session + api-key secrets are **sha256** (not argon2), shared `extractBearerToken` + helper. Matches. + +--- + +## 7. Doc-hygiene notes for REDESIGN.md + +If REDESIGN.md is meant to track reality, these lines are now stale: + +- Move `core.session` / `core.oauth_identity` / `core.oauth_flow` out of the + `core` section and document the separate `auth` schema (better-auth shape). +- `core.api_key`: reconcile "global credential" with the space-scoped + `me.....` key actually shipped. +- §"Private Areas" / §"me space create": the `home`/`share`/`~` convention and the + creator's `owner@home`+`owner@share` (not `owner@root`) grant are shipped, not + deferred — update the V1 scope. +- §"Space": `core.space` does not yet carry embedding or placement columns + (there's a TODO); embedding is hardcoded uniform. Either implement or downgrade + the prose to "future." +- §"Authorization Boundary": rename `effective_tree_access(_space_id, + _principal_id) returns table` to the real `build_tree_access(_member_id, + _space_id) returns jsonb`. +- Command list: `space alter`→`rename`, `apikey revoke`→`delete`, `agent group + list`→`agent groups`; add `me agent add`; `me memory copy` and `me user group + list` are unbuilt; the single-API framing should mention the two RPC endpoints + + `/api/v1/auth/*`. +- Add the **last-admin safeguard** to a "not yet implemented" list so it isn't + assumed to exist. From b5246a43542ecf06f9b6f63831b2528570e67df0 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 14:18:18 +0200 Subject: [PATCH 093/156] feat(api-key): make API keys global, not space-scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keys are now global per-principal credentials. The string drops the space slug (me..), and apiKey.* moves from the memory RPC to the session-only user RPC, gated by agent ownership. One key authenticates into any space the agent has been admitted to — the space comes from X-Me-Space and the build_tree_access empty-set gate (a key with no access there gets 403, not a parse-time 400). No DB migration: core.api_key was already space_id-free. Because the key no longer carries a space, `me install`, `me mcp`, and the Claude plugin now require an explicit space (--space / ME_SPACE / active space; the plugin gains a required `space` config). Minting a key no longer requires the agent to be in a space first. Legacy 4-part keys (me...) are rejected with a clear "recreate it with `me apikey create`" message — server-side (401 LEGACY_API_KEY, surfaced through CLI commands and MCP tool calls) and as a fail-fast at `me mcp` startup. Docs: CLAUDE.md, the claude-plugin README, and REDESIGN_DIFFERENCES.md updated (the api-key divergence is removed, since it now matches the redesign); TODO.md notes the global-key model for the docs rewrite. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 8 +- REDESIGN_DIFFERENCES.md | 56 ++----- TODO.md | 23 +++ .../claude-plugin/.claude-plugin/plugin.json | 6 + packages/claude-plugin/.mcp.json | 4 +- packages/claude-plugin/README.md | 5 +- packages/cli/claude/capture.test.ts | 28 +++- packages/cli/claude/capture.ts | 13 +- packages/cli/commands/apikey.ts | 44 ++--- packages/cli/commands/claude.ts | 5 + packages/cli/commands/codex.ts | 5 + packages/cli/commands/gemini.ts | 5 + packages/cli/commands/mcp.test.ts | 27 +++ packages/cli/commands/mcp.ts | 40 +++-- packages/cli/commands/opencode.ts | 5 + packages/cli/mcp/agent-install.ts | 23 ++- packages/cli/mcp/install.test.ts | 12 +- packages/cli/mcp/install.ts | 22 ++- packages/client/index.ts | 8 +- packages/client/memory.ts | 25 +-- packages/client/user.ts | 27 ++- packages/engine/core/api-key.test.ts | 56 ++++--- packages/engine/core/api-key.ts | 41 +++-- packages/engine/core/db.integration.test.ts | 4 +- packages/engine/core/index.ts | 1 + packages/protocol/space/index.ts | 25 +-- packages/protocol/{space => user}/api-key.ts | 9 +- packages/protocol/user/index.ts | 24 ++- .../authenticate-space.integration.test.ts | 75 +++++++-- .../server/middleware/authenticate-space.ts | 40 ++--- packages/server/rpc/memory/index.ts | 6 +- .../rpc/memory/management.integration.test.ts | 52 +----- packages/server/rpc/memory/support.ts | 38 +---- packages/server/rpc/user/agent.ts | 8 +- .../rpc/user/api-key.integration.test.ts | 154 ++++++++++++++++++ .../server/rpc/{memory => user}/api-key.ts | 59 ++++--- packages/server/rpc/user/index.ts | 9 +- 37 files changed, 628 insertions(+), 364 deletions(-) create mode 100644 packages/cli/commands/mcp.test.ts rename packages/protocol/{space => user}/api-key.ts (84%) create mode 100644 packages/server/rpc/user/api-key.integration.test.ts rename packages/server/rpc/{memory => user}/api-key.ts (55%) diff --git a/CLAUDE.md b/CLAUDE.md index c2ee9c4..a0619f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,10 +31,10 @@ Read the relevant docs before starting work on a subsystem. - **Access**: no RLS. `core.build_tree_access(principalId, spaceId)` produces a `_tree_access` jsonb (rows of `tree_path` + `access`) passed into the space SQL functions (`search_memory`, `get_memory`, …). Three additive levels: **1 = read, 2 = write, 3 = owner**; `owner@root` (the empty ltree path) owns the whole space, and an owner grant at any path delegates access-management within that subtree. Two axes: **structural** authority (`principal_space.admin` — roster mutations, groups, invitations) vs **data** authority (owner@path); an admin may also grant data and can self-grant `owner@root`. The auth gate is a non-empty `build_tree_access` (every member holds ≥1 grant). - **Tree conventions**: two reserved roots — per-member `home.` (`~` is input sugar for it; a joining **user** is granted `owner@home`) and the shared `share`. A space **creator** gets `admin` + `owner@home` + `owner@share`, **not** `owner@root` — so it sees `share` and its own `~` but not other members' homes (as an admin it can self-grant `owner@root`). A bare `memory.create` (no `tree`) defaults to `share` (`SHARE_NAMESPACE`). - **API**: JSON-RPC 2.0 over HTTP, two endpoints: - - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `invite.*`, `apiKey.*`). - - `/api/v1/user/rpc` — session only (an api key never authenticates here; agents can't manage agents). `whoami`, `agent.*`, `space.*`. + - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `invite.*`). + - `/api/v1/user/rpc` — session only (an api key never authenticates here; agents can't manage agents). `whoami`, `agent.*`, `apiKey.*`, `space.*`. - Plus REST OAuth device-flow endpoints under `/api/v1/auth/*`. -- **Auth**: humans use a **session token** (OAuth device flow, GitHub/Google); agents use an **api key** (`me...`). Session + api-key secrets are sha256 (compared by equality in SQL), not argon2. +- **Auth**: humans use a **session token** (OAuth device flow, GitHub/Google); agents use an **api key** (`me..`). Api keys are **global** per-principal credentials, not space-bound: the same key works in any space the agent has been admitted to (the space comes from `X-Me-Space`, gated by `build_tree_access`). Session + api-key secrets are sha256 (compared by equality in SQL), not argon2. - **Embedding**: Vercel AI SDK; OpenAI `text-embedding-3-small` (1536-dim) in production; Ollama supported for local dev. - **CLI**: `me` binary — `login`, `logout`, `whoami`, `space`, `group`, `access`, `agent`, `apikey`, `memory` (+ top-level aliases like `me search`, `me create`), `mcp`, `claude`/`codex`/`gemini`/`opencode`, `serve`, `pack`. @@ -42,7 +42,7 @@ Read the relevant docs before starting work on a subsystem. - **Principal** = the union **user | agent | group** (`principal.kind` = `'u'` | `'a'` | `'g'`). The space roster (`principal_space`) holds principals. `principal.member_id` is a generated column equal to `id` for users/agents (NOT groups). - **Member** = the **user/agent** sense only — group members and api-key holders. So params split as `principalId` (roster / grants, any kind) vs `memberId` (group membership, api keys; u|a only). The space-roster surface is principal-centric (`principal.*` methods, `SpacePrincipal` type), reserving "member" for u|a. -- **Space**: identified by an immutable 12-char `slug` (which is the `me_` schema name, the api-key prefix, and the `X-Me-Space` value) and a renamable `name`. `me space rename` changes only the name. No org / engine / shard concepts. +- **Space**: identified by an immutable 12-char `slug` (which is the `me_` schema name and the `X-Me-Space` value) and a renamable `name`. `me space rename` changes only the name. No org / engine / shard concepts. - **Admin**: `principal_space.admin` is *structural* authority — roster mutations (`principal.add`/`remove`), groups, and invitations (`invite.*`) — distinct from data ownership (owner@path via `tree_access`). Enumerating the whole roster (`principal.list`) is admin-only; **any member** may `principal.resolve`/`lookup` (a targeted name↔id lookup, not enumeration). Admin transfers **transitively** through a group whose own `principal_space.admin` is true; agents are never admins. - **Transitive membership** (Model 2): a group member gains the group's space membership, its space-admin (if the group is admin), and its tree-access grants. diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index 61f909d..c4bd8e5 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -20,14 +20,13 @@ File references are `path:line` against the repo root. | # | Topic | Redesign says | Implementation does | Severity | |---|-------|---------------|---------------------|----------| | A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | -| B | API keys | **Global** per-principal; one key works across many spaces | **Space-scoped**: format `me...`, validated against `X-Me-Space` | **Major** | -| C | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | -| D | Embedding config | Per-space model/dimension, recorded in `core.space` | Hardcoded `text-embedding-3-small` / 1536 for all spaces; `core.space` has a TODO comment, no such columns | **Major** | -| E | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | -| F | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | -| G | Last-admin safeguard | Must prevent removing/demoting the last admin | **Not implemented** | Gap | -| H | `me memory copy`/`cp` | Listed | **Not implemented** | Gap | -| I | `me user group list` | The one supported user command for v1 | **Not implemented** | Gap | +| B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | +| C | Embedding config | Per-space model/dimension, recorded in `core.space` | Hardcoded `text-embedding-3-small` / 1536 for all spaces; `core.space` has a TODO comment, no such columns | **Major** | +| D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | +| E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | +| F | Last-admin safeguard | Must prevent removing/demoting the last admin | **Not implemented** | Gap | +| G | `me memory copy`/`cp` | Listed | **Not implemented** | Gap | +| H | `me user group list` | The one supported user command for v1 | **Not implemented** | Gap | The agent access-masking model (the part the doc was least confident about) **is** implemented as designed — see §6. @@ -56,28 +55,7 @@ redesign (§"Authorization Boundary") is true for *authorization* (principals, grants, groups) but **not** for *authentication* — authentication is its own schema. The redesign never mentions an `auth` schema or better-auth. -### B. API keys are space-scoped, not global - -The redesign is explicit (§`core.api_key`): "API keys are global credentials for -a principal, not credentials for a specific space… A user's agent can use the -same key across multiple spaces." - -The implementation does the opposite. The key format is -`me...` (`packages/engine/core/api-key.ts:4`, -`packages/server/middleware/authenticate-space.ts:9`), so the **space slug is -baked into the key**. `authenticate-space.ts:114` rejects the request if the -key's slug doesn't match the `X-Me-Space` header. A key therefore authenticates -into exactly one space; using an agent across N spaces requires N keys. - -`core.api_key` (`006_api_key.sql:4`) columns: `id, member_id (→ principal.member_id, -u|a only), lookup_id (unique, 16-char), secret (sha256), name, created_at, -expires_at`, with a unique `(member_id, name)`. Note the table itself has **no -`space_id` column** — the scoping is purely via the slug embedded in the -presented key string, not a DB relationship. This is worth reconciling: the -storage is principal-global (as the redesign wants) but the credential string is -space-bound (as the redesign rejects). - -### C. Reserved tree paths and provisioning are built, not deferred +### B. Reserved tree paths and provisioning are built, not deferred This is the largest behavioral divergence. The redesign's V1 scope says (§"Private Areas", §"me space create"): @@ -106,7 +84,7 @@ respected. But the **convention layer the redesign deferred is shipped**, and th creator's grant is `home`+`share` rather than `root`. Any reader of REDESIGN.md would expect a fresh space to be empty and root-owned; it is neither. -### D. Embedding model/dimension is hardcoded, not per-space +### C. Embedding model/dimension is hardcoded, not per-space The redesign (§"Space") wants the embedding model and dimension to be per-space, templated into the DDL, and recorded in `core.space` so the server can route @@ -135,7 +113,7 @@ aren't there yet). ## 3. Naming / shape divergences (same concept, different surface) -### E. Access resolution function +### D. Access resolution function - Redesign: `core.effective_tree_access(_space_id uuid, _principal_id uuid) returns table(tree_path ltree, access int4)`. @@ -150,15 +128,15 @@ Space functions consume it as a `_tree_access jsonb` argument not a later optimization. Per CLAUDE.md the auth gate is "non-empty `build_tree_access`." -### F. Two RPC endpoints, plus REST auth +### E. Two RPC endpoints, plus REST auth The redesign describes "the hosted API server exposes JSON-RPC over HTTPS" as a single surface. The implementation splits it (`packages/server/router.ts:252`): - `/api/v1/memory/rpc` — session **or** api-key + required `X-Me-Space`. Hosts - `memory.*`, `principal.*`, `group.*`, `grant.*`, `invite.*`, `apiKey.*`. + `memory.*`, `principal.*`, `group.*`, `grant.*`, `invite.*`. - `/api/v1/user/rpc` — **session only** (api keys rejected here). Hosts `whoami`, - `agent.*`, `space.*`. + `agent.*`, `apiKey.*`, `space.*`. - `/api/v1/auth/*` — REST device-flow endpoints (`device/code`, `device/token`, `device/verify`, `device/approve`, `callback/:provider`). @@ -181,13 +159,13 @@ All same-intent, different spelling: ## 4. Not yet implemented (gaps vs. the redesign) -- **G. Last-admin safeguard.** The redesign requires that no cascade or removal +- **F. Last-admin safeguard.** The redesign requires that no cascade or removal may strip the last `principal_space.admin = true` from a space (§"Last-Admin Safeguard", §`core.principal_space`). No such check exists in SQL or app code. -- **H. `me memory copy` / `cp`.** Listed in §"Memory Commands"; `move`/`mv` +- **G. `me memory copy` / `cp`.** Listed in §"Memory Commands"; `move`/`mv` exists, `copy`/`cp` does not (`commands/memory.ts`). The MCP server likewise has `me_memory_mv` but no copy tool. -- **I. `me user group list `.** The redesign names this the single +- **H. `me user group list `.** The redesign names this the single supported `me user` command for v1; there is **no `me user` command** at all. - **Verified-email enforcement on invite acceptance.** The redesign (§`core.space_invitation`) wants acceptance to require an OAuth-verified email matching the invitation @@ -285,8 +263,6 @@ If REDESIGN.md is meant to track reality, these lines are now stale: - Move `core.session` / `core.oauth_identity` / `core.oauth_flow` out of the `core` section and document the separate `auth` schema (better-auth shape). -- `core.api_key`: reconcile "global credential" with the space-scoped - `me.....` key actually shipped. - §"Private Areas" / §"me space create": the `home`/`share`/`~` convention and the creator's `owner@home`+`owner@share` (not `owner@root`) grant are shipped, not deferred — update the V1 scope. diff --git a/TODO.md b/TODO.md index 349f844..439835a 100644 --- a/TODO.md +++ b/TODO.md @@ -14,6 +14,10 @@ that raw NOT_FOUND, so the user has to know to run `me agent add ` first. 'me agent add ' first"). Added a reusable `isAppErrorCode` helper to util.ts. (Auto-adding the agent was considered but skipped — silently changing space membership as a side effect of minting a key is surprising.) +- [x] Superseded (2026-06-05) — by the "API keys are global" change. `apiKey.*` + moved to the user RPC and minting now needs only agent ownership (no active + space, no `me agent add` prerequisite), so the NOT_FOUND mapping above was + removed from `me apikey create`. (`isAppErrorCode` stays in util.ts.) ## Space invitations: email delivery + expiry (deferred from v1) @@ -174,6 +178,25 @@ now the authoritative summary of the current design. `me space/group/access/agent/apikey/memory` command surface. Update `docs/cli/*` (drop engine/org/invitation/user/owner/role; add space/group/access/agent) and `docs/mcp/*`. The docs-site renders these. +- [ ] **API keys are global** (changed 2026-06-05) — document the model when the + docs are rewritten: + - Format is `me..` (no space slug). One key works in + **any** space the owning agent has been admitted to; the space is chosen + per-request via `X-Me-Space`. + - `apiKey.*` lives on the **user RPC** (session-only) — TS consumers call + `createUserClient().apiKey.*`, not the memory client. `me apikey` needs + no active space. + - Minting a key requires only **agent ownership** — not space membership. + A freshly-minted key is inert until the agent is added to a space + (`me agent add`) and granted access there. + - **Owner masking caveat**: an agent's effective access is capped by its + owner's current access, so a key authenticates into a space only if the + *owner* also has access there (document this, it surprises people). + - `me install`, `me mcp`, and the Claude plugin now require an + explicit space (`--space` / `ME_SPACE` / active space; the plugin has a + new required `space` config) since the key no longer carries it. + - Migration note for any existing setups: old 4-part `me.....` keys + are invalid and must be re-minted. ## Deploy: env rename coordination (Phase 5) diff --git a/packages/claude-plugin/.claude-plugin/plugin.json b/packages/claude-plugin/.claude-plugin/plugin.json index 63d7eca..7bf50b3 100644 --- a/packages/claude-plugin/.claude-plugin/plugin.json +++ b/packages/claude-plugin/.claude-plugin/plugin.json @@ -24,6 +24,12 @@ "default": "https://api.memory.build", "required": true }, + "space": { + "type": "string", + "title": "Space", + "description": "Space slug to store memories in (the X-Me-Space). API keys are global, so the space must be set explicitly.", + "required": true + }, "tree_prefix": { "type": "string", "title": "Tree prefix", diff --git a/packages/claude-plugin/.mcp.json b/packages/claude-plugin/.mcp.json index 677100c..c518cbf 100644 --- a/packages/claude-plugin/.mcp.json +++ b/packages/claude-plugin/.mcp.json @@ -7,7 +7,9 @@ "--server", "${user_config.server}", "--api-key", - "${user_config.api_key}" + "${user_config.api_key}", + "--space", + "${user_config.space}" ] } } diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index ef6fff7..7aad5c5 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -69,7 +69,7 @@ claude plugin install memory-engine@memory-engine --scope local # this repo, ## Configure -The plugin needs three values: `api_key`, `server`, and `tree_prefix`. Claude Code does not prompt for them at install time — you configure them from inside a session. +The plugin needs four values: `api_key`, `server`, `space`, and `tree_prefix`. Claude Code does not prompt for them at install time — you configure them from inside a session. ```text claude # start a session @@ -77,6 +77,7 @@ claude # start a session # → Installed → memory-engine → Configure # → api_key (sensitive — stored in keychain) # → server (default https://api.memory.build) +# → space (the space slug — api keys are global, so this is required) # → tree_prefix (default claude_code.sessions) # → values take effect immediately; no restart required ``` @@ -143,7 +144,7 @@ Claude Code handles the cleanup. Your captured memories and API keys are preserv The hook ran but userConfig isn't filled in. Open `/plugin → memory-engine → Configure` and set the api_key. **`Plugin option "X" isn't set` in Claude Code's error panel** -A required userConfig value is missing for either a hook or the MCP server. Configure all three: api_key, server, tree_prefix. +A required userConfig value is missing for either a hook or the MCP server. Configure all four: api_key, server, space, tree_prefix. **Hook fires but no memories appear** - Confirm the api_key is valid: diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index c14d7df..d17a086 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -23,7 +23,7 @@ const BASE_EVENT = { const CONFIG: HookConfig = { server: "https://api.example.com", - apiKey: "me.eng123.aaa.bbb", + apiKey: "me.lookupid12345678.secret", space: "eng123", treePrefix: "claude_code.sessions", }; @@ -246,15 +246,23 @@ describe("resolveHookConfigFromEnv", () => { expect(cfg).toBeNull(); }); - test("returns config when api_key is present", () => { + test("returns null when space is missing (keys are global)", () => { const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.eng.aaa.bbb", + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + }); + expect(cfg).toBeNull(); + }); + + test("returns config when api_key and space are present", () => { + const cfg = resolveHookConfigFromEnv({ + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", CLAUDE_PLUGIN_OPTION_SERVER: "https://api.example.com", CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "my.prefix", }); expect(cfg).toEqual({ - apiKey: "me.eng.aaa.bbb", - space: "eng", + apiKey: "me.lookupid12345678.secret", + space: "eng123def456", server: "https://api.example.com", treePrefix: "my.prefix", }); @@ -262,11 +270,12 @@ describe("resolveHookConfigFromEnv", () => { test("falls back to default server and tree_prefix", () => { const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.eng.aaa.bbb", + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", }); expect(cfg).toEqual({ - apiKey: "me.eng.aaa.bbb", - space: "eng", + apiKey: "me.lookupid12345678.secret", + space: "eng123def456", server: "https://api.memory.build", treePrefix: "claude_code.sessions", }); @@ -274,7 +283,8 @@ describe("resolveHookConfigFromEnv", () => { test("treats empty string as missing (falls back to default)", () => { const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.eng.aaa.bbb", + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", CLAUDE_PLUGIN_OPTION_SERVER: "", CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "", }); diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index 3daa892..4ff6107 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -18,19 +18,12 @@ export interface HookConfig { server: string; /** Agent api key (from the plugin's sensitive userConfig). */ apiKey: string; - /** Active space slug (X-Me-Space); defaults to the api key's own slug. */ + /** Active space slug (X-Me-Space). */ space: string; /** Tree path prefix for captured memories (ltree). */ treePrefix: string; } -/** Extract the space slug embedded in an api key (`me...`). */ -export function slugFromApiKey(apiKey: string): string | undefined { - if (!apiKey.startsWith("me.")) return undefined; - const parts = apiKey.split("."); - return parts.length >= 4 ? parts[1] : undefined; -} - // ============================================================================= // Event types // ============================================================================= @@ -175,8 +168,8 @@ export function resolveHookConfigFromEnv( const apiKey = env.CLAUDE_PLUGIN_OPTION_API_KEY; if (!apiKey) return null; - // The space defaults to the api key's own slug; an explicit env var overrides. - const space = env.CLAUDE_PLUGIN_OPTION_SPACE || slugFromApiKey(apiKey); + // Api keys are global, so the space must be configured explicitly. + const space = env.CLAUDE_PLUGIN_OPTION_SPACE; if (!space) return null; return { diff --git a/packages/cli/commands/apikey.ts b/packages/cli/commands/apikey.ts index 3edd3ce..804bd61 100644 --- a/packages/cli/commands/apikey.ts +++ b/packages/cli/commands/apikey.ts @@ -1,8 +1,8 @@ /** - * me apikey — manage an agent's API keys in the active space. + * me apikey — manage your agents' API keys. * - * Keys are agent-only (humans authenticate with a session). The agent must - * already be in the space (see `me agent add`). The plaintext key is shown + * Keys are agent-only (humans authenticate with a session) and global: a key + * works in any space the agent has been admitted to. The plaintext key is shown * exactly once, by `create`. There is no revoke state — delete is the removal. * * - me apikey create [name] [--expires ]: mint a key (shown once) @@ -17,18 +17,15 @@ import { Command } from "commander"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; import { - buildMemoryClient, buildUserClient, handleError, - isAppErrorCode, requireSession, - requireSpace, resolveAgentId, } from "../util.ts"; function createApiKeyCreateCommand(): Command { return new Command("create") - .description("mint an API key for an agent in the active space") + .description("mint an API key for one of your agents") .argument("", "agent id or name") .argument("[name]", "key name (auto-generated if omitted)") .option("--expires ", "expiration timestamp (ISO 8601)") @@ -37,15 +34,13 @@ function createApiKeyCreateCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireSpace(creds, fmt); const user = buildUserClient(creds); - const memory = buildMemoryClient(creds); const keyName = name ?? `cli-${new Date().toISOString().slice(0, 10)}`; try { const agentId = await resolveAgentId(user, agent, fmt); - const result = await memory.apiKey.create({ + const result = await user.apiKey.create({ agentId, name: keyName, expiresAt: opts.expires ?? null, @@ -58,21 +53,10 @@ function createApiKeyCreateCommand(): Command { "API key — save it now; it won't be shown again", ); clack.log.info( - "Give it to the agent via ME_API_KEY or its MCP config.", + "Give it to the agent via ME_API_KEY or its MCP config. It works in any space the agent is a member of.", ); }); } catch (error) { - // apiKey.create requires the agent to already be in the space; surface - // the prerequisite instead of a bare NOT_FOUND. - if (isAppErrorCode(error, "NOT_FOUND")) { - const msg = `Agent '${agent}' isn't in this space yet — run 'me agent add ${agent}' first, then 'me apikey create' again.`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg, code: "NOT_FOUND" }, fmt, () => {}); - } - process.exit(1); - } handleError(error, fmt, { sessionServer: creds.server }); } }); @@ -88,13 +72,11 @@ function createApiKeyListCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireSpace(creds, fmt); const user = buildUserClient(creds); - const memory = buildMemoryClient(creds); try { const agentId = await resolveAgentId(user, agent, fmt); - const { apiKeys } = await memory.apiKey.list({ memberId: agentId }); + const { apiKeys } = await user.apiKey.list({ memberId: agentId }); output({ apiKeys }, fmt, () => { if (apiKeys.length === 0) { console.log(" No API keys."); @@ -120,11 +102,10 @@ function createApiKeyGetCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireSpace(creds, fmt); - const memory = buildMemoryClient(creds); + const user = buildUserClient(creds); try { - const { apiKey } = await memory.apiKey.get({ id }); + const { apiKey } = await user.apiKey.get({ id }); output({ apiKey }, fmt, () => { if (!apiKey) { clack.log.warn("API key not found."); @@ -153,7 +134,6 @@ function createApiKeyDeleteCommand(): Command { const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireSpace(creds, fmt); if (fmt === "text" && !opts.yes) { const confirmed = await clack.confirm({ @@ -166,9 +146,9 @@ function createApiKeyDeleteCommand(): Command { } } - const memory = buildMemoryClient(creds); + const user = buildUserClient(creds); try { - const result = await memory.apiKey.delete({ id }); + const result = await user.apiKey.delete({ id }); output({ id, ...result }, fmt, () => { if (result.deleted) clack.log.success("API key deleted."); else clack.log.warn("API key not found."); @@ -181,7 +161,7 @@ function createApiKeyDeleteCommand(): Command { export function createApiKeyCommand(): Command { const apikey = new Command("apikey").description( - "manage agent API keys in the active space", + "manage your agents' API keys", ); apikey.addCommand(createApiKeyCreateCommand()); apikey.addCommand(createApiKeyListCommand()); diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 0bd41d0..25f1993 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -56,6 +56,10 @@ function createClaudeInstallCommand(): Command { .description("register me as an MCP server with Claude Code") .option("--api-key ", "API key to embed in MCP config") .option("--server ", "server URL to embed in MCP config") + .option( + "--space ", + "space to embed in MCP config (else ME_SPACE / active space)", + ) .option( "-s, --scope ", `Claude Code config scope (${CLAUDE_SCOPES.join(", ")})`, @@ -71,6 +75,7 @@ function createClaudeInstallCommand(): Command { await runAgentMcpInstall("claude", { apiKey: opts.apiKey, server: globalOpts.server ?? opts.server, + space: opts.space, scope: opts.scope, }); }, diff --git a/packages/cli/commands/codex.ts b/packages/cli/commands/codex.ts index 51986b1..dc3c4af 100644 --- a/packages/cli/commands/codex.ts +++ b/packages/cli/commands/codex.ts @@ -16,11 +16,16 @@ function createCodexInstallCommand(): Command { .description("register me as an MCP server with Codex CLI") .option("--api-key ", "API key to embed in MCP config") .option("--server ", "server URL to embed in MCP config") + .option( + "--space ", + "space to embed in MCP config (else ME_SPACE / active space)", + ) .action(async (opts: AgentInstallOptions, cmd: Command) => { const globalOpts = cmd.optsWithGlobals(); await runAgentMcpInstall("codex", { apiKey: opts.apiKey, server: globalOpts.server ?? opts.server, + space: opts.space, }); }); } diff --git a/packages/cli/commands/gemini.ts b/packages/cli/commands/gemini.ts index 6a51d78..d16d938 100644 --- a/packages/cli/commands/gemini.ts +++ b/packages/cli/commands/gemini.ts @@ -26,6 +26,10 @@ function createGeminiInstallCommand(): Command { .description("register me as an MCP server with Gemini CLI") .option("--api-key ", "API key to embed in MCP config") .option("--server ", "server URL to embed in MCP config") + .option( + "--space ", + "space to embed in MCP config (else ME_SPACE / active space)", + ) .option( "-s, --scope ", `Gemini CLI config scope (${GEMINI_SCOPES.join(", ")})`, @@ -41,6 +45,7 @@ function createGeminiInstallCommand(): Command { await runAgentMcpInstall("gemini", { apiKey: opts.apiKey, server: globalOpts.server ?? opts.server, + space: opts.space, scope: opts.scope, }); }, diff --git a/packages/cli/commands/mcp.test.ts b/packages/cli/commands/mcp.test.ts new file mode 100644 index 0000000..9100805 --- /dev/null +++ b/packages/cli/commands/mcp.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from "bun:test"; +import { isLegacyApiKey } from "./mcp.ts"; + +// Guards the CLI's copy of the legacy-key detector (duplicated from +// @memory.build/engine/core to avoid an engine dependency). Keep in sync with +// the engine version's tests. +describe("isLegacyApiKey", () => { + const legacy = `me.abc123def456.lookupid12345678.${"s".repeat(32)}`; + + test("true for a 4-part legacy (space-scoped) key", () => { + expect(isLegacyApiKey(legacy)).toBe(true); + }); + + test("false for a current 3-part key", () => { + expect(isLegacyApiKey(`me.lookupid12345678.${"s".repeat(32)}`)).toBe(false); + }); + + test("false for an opaque session-like token", () => { + expect(isLegacyApiKey("a".repeat(43))).toBe(false); + }); + + test("false for a 4-part token with a malformed slug", () => { + expect( + isLegacyApiKey(`me.BADSLUG78901.lookupid12345678.${"s".repeat(32)}`), + ).toBe(false); + }); +}); diff --git a/packages/cli/commands/mcp.ts b/packages/cli/commands/mcp.ts index 7561b16..4224e61 100644 --- a/packages/cli/commands/mcp.ts +++ b/packages/cli/commands/mcp.ts @@ -4,11 +4,11 @@ * Authenticates to a space with either a human session (from `me login`) or an * agent api key, and targets the active space (the X-Me-Space). Resolution: * - token: --api-key > ME_API_KEY > stored session token - * - space: --space > ME_SPACE > stored active space > the api key's own slug + * - space: --space > ME_SPACE > stored active space * * The common case is a logged-in human: `me mcp` just works against the active - * space. Agents pass ME_API_KEY (the key embeds its space, so --space is - * optional). + * space. Agents pass ME_API_KEY (keys are global, so a space must be given via + * --space / ME_SPACE — the installers bake it in). * * MCP registration with individual AI tools lives in per-agent commands: * me opencode install, me gemini install, me codex install @@ -18,11 +18,21 @@ import { Command } from "commander"; import { resolveCredentials } from "../credentials.ts"; import { runMcpServer } from "../mcp/server.ts"; -/** Extract the space slug embedded in an api key (`me...`). */ -function slugFromApiKey(token: string): string | undefined { - if (!token.startsWith("me.")) return undefined; +/** + * True if the token is a legacy 4-part api key (`me...`), + * the retired space-scoped format that no longer authenticates. Duplicated from + * `@memory.build/engine/core`'s `isLegacyApiKey` so the CLI doesn't depend on the + * engine package; the legacy format is frozen, so this won't drift. + */ +export function isLegacyApiKey(token: string): boolean { const parts = token.split("."); - return parts.length >= 4 ? parts[1] : undefined; + return ( + parts.length === 4 && + parts[0] === "me" && + /^[a-z0-9]{12}$/.test(parts[1] ?? "") && + /^[A-Za-z0-9_-]{16}$/.test(parts[2] ?? "") && + (parts[3]?.length ?? 0) === 32 + ); } function createMcpRunAction() { @@ -40,11 +50,17 @@ function createMcpRunAction() { process.exit(1); } - // Space: --space > ME_SPACE / stored active space > the api key's own slug. - const space = - (opts.space as string | undefined) ?? - creds.activeSpace ?? - slugFromApiKey(token); + // Fail fast on a retired space-scoped key rather than starting the server and + // failing on the first tool call with a server-side error. + if (isLegacyApiKey(token)) { + console.error( + "Error: this API key uses the old space-scoped format (me...) and no longer works. Recreate it with 'me apikey create ', then update ME_API_KEY or your MCP config.", + ); + process.exit(1); + } + + // Space: --space > ME_SPACE / stored active space. + const space = (opts.space as string | undefined) ?? creds.activeSpace; if (!space) { console.error( "Error: no active space. Run 'me space use ', or pass --space / set ME_SPACE.", diff --git a/packages/cli/commands/opencode.ts b/packages/cli/commands/opencode.ts index 308fd71..849a8c5 100644 --- a/packages/cli/commands/opencode.ts +++ b/packages/cli/commands/opencode.ts @@ -16,11 +16,16 @@ function createOpenCodeInstallCommand(): Command { .description("register me as an MCP server with OpenCode") .option("--api-key ", "API key to embed in MCP config") .option("--server ", "server URL to embed in MCP config") + .option( + "--space ", + "space to embed in MCP config (else ME_SPACE / active space)", + ) .action(async (opts: AgentInstallOptions, cmd: Command) => { const globalOpts = cmd.optsWithGlobals(); await runAgentMcpInstall("opencode", { apiKey: opts.apiKey, server: globalOpts.server ?? opts.server, + space: opts.space, }); }); } diff --git a/packages/cli/mcp/agent-install.ts b/packages/cli/mcp/agent-install.ts index 7ac5dd7..4bae7f6 100644 --- a/packages/cli/mcp/agent-install.ts +++ b/packages/cli/mcp/agent-install.ts @@ -11,6 +11,8 @@ import { buildMeCommand, installMcpServer, MCP_TOOLS } from "./install.ts"; export interface AgentInstallOptions { apiKey?: string; server?: string; + /** The space slug to bake into the MCP command (api keys are global). */ + space?: string; /** * Configuration scope for tools that support it (Claude Code, Gemini CLI). * Ignored by tools without a scope concept (Codex, OpenCode). @@ -34,14 +36,16 @@ export async function runAgentMcpInstall( process.exit(1); } - // Resolve credentials: flags > env (ME_API_KEY) > server default. MCP configs - // bake in a long-lived agent api key (a human session would expire), so an - // api key is required here — mint one with `me apikey create `. - let { apiKey, server } = opts; - if (!apiKey || !server) { + // Resolve credentials: flags > env (ME_API_KEY / ME_SPACE) > stored config. + // MCP configs bake in a long-lived agent api key (a human session would + // expire), so an api key is required here — mint one with + // `me apikey create `. Keys are global, so a space must be baked in too. + let { apiKey, server, space } = opts; + if (!apiKey || !server || !space) { const creds = resolveCredentials(server); if (!apiKey) apiKey = creds.apiKey; if (!server) server = creds.server; + if (!space) space = creds.activeSpace; } if (!apiKey) { @@ -56,6 +60,13 @@ export async function runAgentMcpInstall( process.exit(1); } + if (!space) { + clack.log.error( + "No space available. Pass --space, set ME_SPACE, or run 'me space use '.", + ); + process.exit(1); + } + // For CLI tools, require the binary to be on PATH. JSON-file tools // (e.g. OpenCode) just edit a config file and don't need the binary. if (tool.method === "cli" && Bun.which(tool.bin) === null) { @@ -66,7 +77,7 @@ export async function runAgentMcpInstall( } // Build the me mcp command with baked-in credentials - const meCmd = buildMeCommand(apiKey, server); + const meCmd = buildMeCommand(apiKey, server, space); const spin = clack.spinner(); spin.start(`Registering with ${tool.name}...`); diff --git a/packages/cli/mcp/install.test.ts b/packages/cli/mcp/install.test.ts index 2e59640..1ffe47d 100644 --- a/packages/cli/mcp/install.test.ts +++ b/packages/cli/mcp/install.test.ts @@ -6,13 +6,17 @@ import { buildMeCommand, buildOpenCodeConfig, MCP_TOOLS } from "./install.ts"; describe("buildMeCommand", () => { test("uses bare 'me' command on PATH", () => { - const cmd = buildMeCommand("test-key-123", "https://api.memory.build"); + const cmd = buildMeCommand( + "test-key-123", + "https://api.memory.build", + "abc123def456", + ); expect(cmd[0]).toBe("me"); expect(cmd[1]).toBe("mcp"); }); - test("includes --api-key and --server with correct values", () => { - const cmd = buildMeCommand("k", "https://example.com"); + test("includes --api-key, --server, and --space with correct values", () => { + const cmd = buildMeCommand("k", "https://example.com", "abc123def456"); expect(cmd).toEqual([ "me", "mcp", @@ -20,6 +24,8 @@ describe("buildMeCommand", () => { "k", "--server", "https://example.com", + "--space", + "abc123def456", ]); }); }); diff --git a/packages/cli/mcp/install.ts b/packages/cli/mcp/install.ts index a2a6abb..cf450b0 100644 --- a/packages/cli/mcp/install.ts +++ b/packages/cli/mcp/install.ts @@ -116,11 +116,25 @@ export function detectInstalledTools(): McpTool[] { /** * Build the `me mcp` command array with baked-in credentials. * - * Always uses bare `me` — the binary is expected to be on PATH - * whether installed via the install script, Homebrew, or npm. + * Api keys are global, so the space must be baked in explicitly (`--space`). + * Always uses bare `me` — the binary is expected to be on PATH whether installed + * via the install script, Homebrew, or npm. */ -export function buildMeCommand(apiKey: string, serverUrl: string): string[] { - return ["me", "mcp", "--api-key", apiKey, "--server", serverUrl]; +export function buildMeCommand( + apiKey: string, + serverUrl: string, + space: string, +): string[] { + return [ + "me", + "mcp", + "--api-key", + apiKey, + "--server", + serverUrl, + "--space", + space, + ]; } // ============================================================================= diff --git a/packages/client/index.ts b/packages/client/index.ts index b9e3e23..d015b53 100644 --- a/packages/client/index.ts +++ b/packages/client/index.ts @@ -5,10 +5,10 @@ * * - {@link createMemoryClient} — space data-plane + management. * Talks to /api/v1/memory/rpc with the active space carried as X-Me-Space. - * Memory CRUD/search plus principal/group/grant/apiKey management. + * Memory CRUD/search plus principal/group/grant/invite management. * * - {@link createUserClient} — session-only, user-scoped. - * Talks to /api/v1/user/rpc: whoami, agent lifecycle, space discovery. + * Talks to /api/v1/user/rpc: whoami, agent lifecycle, api keys, space discovery. * * - {@link createAuthClient} — auth client (no auth). * OAuth device flow for CLI login. Returns a session token. @@ -45,7 +45,6 @@ export { createAuthClient, DeviceFlowError } from "./auth.ts"; export { isRpcError, RpcError } from "./errors.ts"; // Memory client (space data-plane + management) export { - type ApiKeyNamespace, createMemoryClient, type GrantNamespace, type GroupNamespace, @@ -55,9 +54,10 @@ export { type MemoryNamespace, type PrincipalNamespace, } from "./memory.ts"; -// User client (session-only: whoami, agent lifecycle, space discovery) +// User client (session-only: whoami, agent lifecycle, api keys, space discovery) export { type AgentNamespace, + type ApiKeyNamespace, createUserClient, type SpaceNamespace, type UserClient, diff --git a/packages/client/memory.ts b/packages/client/memory.ts index 76a0725..377c4af 100644 --- a/packages/client/memory.ts +++ b/packages/client/memory.ts @@ -3,7 +3,8 @@ * * Talks to POST /api/v1/memory/rpc, authenticated by a session token (human) or * an api key (agent), with the active space selected via the X-Me-Space header. - * Namespaces: memory (data plane) + principal / group / grant / apiKey (management). + * Namespaces: memory (data plane) + principal / group / grant / invite (management). + * (Agent lifecycle and api keys live on the user client.) * * @example * ```ts @@ -35,14 +36,6 @@ import type { MemoryUpdateParams, } from "@memory.build/protocol/memory"; import type { - ApiKeyCreateParams, - ApiKeyCreateResult, - ApiKeyDeleteParams, - ApiKeyDeleteResult, - ApiKeyGetParams, - ApiKeyGetResult, - ApiKeyListParams, - ApiKeyListResult, GrantListParams, GrantListResult, GrantRemoveParams, @@ -153,20 +146,12 @@ export interface InviteNamespace { revoke(params: InviteRevokeParams): Promise; } -export interface ApiKeyNamespace { - create(params: ApiKeyCreateParams): Promise; - list(params: ApiKeyListParams): Promise; - get(params: ApiKeyGetParams): Promise; - delete(params: ApiKeyDeleteParams): Promise; -} - export interface MemoryClient { memory: MemoryNamespace; principal: PrincipalNamespace; group: GroupNamespace; grant: GrantNamespace; invite: InviteNamespace; - apiKey: ApiKeyNamespace; /** Update the bearer token (session or api key) at runtime. */ setToken(token: string): void; @@ -236,12 +221,6 @@ export function createMemoryClient( list: (p) => rpc("invite.list", p ?? {}), revoke: (p) => rpc("invite.revoke", p), }, - apiKey: { - create: (p) => rpc("apiKey.create", p), - list: (p) => rpc("apiKey.list", p), - get: (p) => rpc("apiKey.get", p), - delete: (p) => rpc("apiKey.delete", p), - }, setToken(token: string) { config.token = token; }, diff --git a/packages/client/user.ts b/packages/client/user.ts index 4757afd..4c200cf 100644 --- a/packages/client/user.ts +++ b/packages/client/user.ts @@ -2,8 +2,9 @@ * User client — session-only, user-scoped operations. * * Talks to POST /api/v1/user/rpc, authenticated by a session token. Namespaces: - * agent (a user's global service accounts) and space (discover/create/manage the - * user's spaces — used by the CLI to pick the active X-Me-Space). + * agent (a user's global service accounts), apiKey (those agents' global keys), + * and space (discover/create/manage the user's spaces — used by the CLI to pick + * the active X-Me-Space). */ import type { AgentCreateParams, @@ -14,6 +15,14 @@ import type { AgentListResult, AgentRenameParams, AgentRenameResult, + ApiKeyCreateParams, + ApiKeyCreateResult, + ApiKeyDeleteParams, + ApiKeyDeleteResult, + ApiKeyGetParams, + ApiKeyGetResult, + ApiKeyListParams, + ApiKeyListResult, SpaceCreateParams, SpaceCreateResult, SpaceDeleteParams, @@ -49,6 +58,13 @@ export interface AgentNamespace { delete(params: AgentDeleteParams): Promise; } +export interface ApiKeyNamespace { + create(params: ApiKeyCreateParams): Promise; + list(params: ApiKeyListParams): Promise; + get(params: ApiKeyGetParams): Promise; + delete(params: ApiKeyDeleteParams): Promise; +} + export interface SpaceNamespace { list(params?: SpaceListParams): Promise; create(params: SpaceCreateParams): Promise; @@ -60,6 +76,7 @@ export interface UserClient { /** The identity behind the session token. */ whoami(params?: WhoamiParams): Promise; agent: AgentNamespace; + apiKey: ApiKeyNamespace; space: SpaceNamespace; /** Update the session token at runtime. */ setToken(token: string): void; @@ -92,6 +109,12 @@ export function createUserClient(options: UserClientOptions = {}): UserClient { rename: (p) => rpc("agent.rename", p), delete: (p) => rpc("agent.delete", p), }, + apiKey: { + create: (p) => rpc("apiKey.create", p), + list: (p) => rpc("apiKey.list", p), + get: (p) => rpc("apiKey.get", p), + delete: (p) => rpc("apiKey.delete", p), + }, space: { list: (p) => rpc("space.list", p ?? {}), create: (p) => rpc("space.create", p), diff --git a/packages/engine/core/api-key.test.ts b/packages/engine/core/api-key.test.ts index 2dccdd6..f6c5ea8 100644 --- a/packages/engine/core/api-key.test.ts +++ b/packages/engine/core/api-key.test.ts @@ -4,6 +4,7 @@ import { generateLookupId, generateSecret, hashApiKeySecret, + isLegacyApiKey, parseApiKey, } from "./api-key"; @@ -49,54 +50,67 @@ describe("hashApiKeySecret", () => { describe("formatApiKey", () => { test("formats key with all parts", () => { - expect( - formatApiKey("abc123def456", "lookupid12345678", "s".repeat(32)), - ).toBe(`me.abc123def456.lookupid12345678.${"s".repeat(32)}`); + expect(formatApiKey("lookupid12345678", "s".repeat(32))).toBe( + `me.lookupid12345678.${"s".repeat(32)}`, + ); }); }); describe("parseApiKey", () => { - const valid = `me.abc123def456.lookupid12345678.${"s".repeat(32)}`; + const valid = `me.lookupid12345678.${"s".repeat(32)}`; test("parses a valid key (round-trips with formatApiKey)", () => { const parsed = parseApiKey(valid); expect(parsed).toEqual({ - spaceSlug: "abc123def456", lookupId: "lookupid12345678", secret: "s".repeat(32), }); if (parsed) { - expect( - formatApiKey(parsed.spaceSlug, parsed.lookupId, parsed.secret), - ).toBe(valid); + expect(formatApiKey(parsed.lookupId, parsed.secret)).toBe(valid); } }); test("returns null for the wrong prefix", () => { - expect( - parseApiKey(`x.abc123def456.lookupid12345678.${"s".repeat(32)}`), - ).toBeNull(); + expect(parseApiKey(`x.lookupid12345678.${"s".repeat(32)}`)).toBeNull(); + }); + + test("returns null for an invalid lookupId", () => { + expect(parseApiKey(`me.short.${"s".repeat(32)}`)).toBeNull(); + }); + + test("returns null for the wrong secret length", () => { + expect(parseApiKey("me.lookupid12345678.tooshort")).toBeNull(); }); - test("returns null for an invalid spaceSlug (uppercase)", () => { + test("returns null for the wrong number of parts", () => { + expect(parseApiKey("me.lookupid12345678")).toBeNull(); + }); + + test("rejects a legacy 4-part key (with space slug)", () => { expect( - parseApiKey(`me.ABC123def456.lookupid12345678.${"s".repeat(32)}`), + parseApiKey(`me.abc123def456.lookupid12345678.${"s".repeat(32)}`), ).toBeNull(); }); +}); + +describe("isLegacyApiKey", () => { + const legacy = `me.abc123def456.lookupid12345678.${"s".repeat(32)}`; - test("returns null for a short spaceSlug", () => { - expect(parseApiKey(`me.abc.lookupid12345678.${"s".repeat(32)}`)).toBeNull(); + test("true for a 4-part legacy (space-scoped) key", () => { + expect(isLegacyApiKey(legacy)).toBe(true); }); - test("returns null for an invalid lookupId", () => { - expect(parseApiKey(`me.abc123def456.short.${"s".repeat(32)}`)).toBeNull(); + test("false for a current 3-part key", () => { + expect(isLegacyApiKey(`me.lookupid12345678.${"s".repeat(32)}`)).toBe(false); }); - test("returns null for the wrong secret length", () => { - expect(parseApiKey("me.abc123def456.lookupid12345678.tooshort")).toBeNull(); + test("false for an opaque session-like token", () => { + expect(isLegacyApiKey("a".repeat(43))).toBe(false); }); - test("returns null for the wrong number of parts", () => { - expect(parseApiKey("me.abc123def456.lookupid12345678")).toBeNull(); + test("false for a 4-part token with a malformed slug", () => { + expect( + isLegacyApiKey(`me.BADSLUG78901.lookupid12345678.${"s".repeat(32)}`), + ).toBe(false); }); }); diff --git a/packages/engine/core/api-key.ts b/packages/engine/core/api-key.ts index 7b76eaf..07dbbce 100644 --- a/packages/engine/core/api-key.ts +++ b/packages/engine/core/api-key.ts @@ -1,12 +1,15 @@ /** * API key helpers for the core control plane. * - * Key format (unchanged from the legacy engine): me.{spaceSlug}.{lookupId}.{secret} + * Key format: me.{lookupId}.{secret} * - me fixed prefix - * - spaceSlug 12-char lowercase alphanumeric (the core.space slug — routing) * - lookupId 16-char id for the indexed db lookup * - secret 32-char base64url random secret * + * Keys are global per-principal credentials, not space-bound: the same key + * authenticates into any space the owning principal has been admitted to (the + * space is selected by the X-Me-Space header, gated by core.build_tree_access). + * * The secret is high-entropy, so we store sha256(secret) and validate by * equality in SQL (core.validate_api_key) — no per-request argon2 verify. This * matches how session tokens are handled (see packages/accounts/util/hash.ts). @@ -43,26 +46,38 @@ export function hashApiKeySecret(secret: string): string { } /** Assemble a full API key string from its parts. */ -export function formatApiKey( - spaceSlug: string, - lookupId: string, - secret: string, -): string { - return `me.${spaceSlug}.${lookupId}.${secret}`; +export function formatApiKey(lookupId: string, secret: string): string { + return `me.${lookupId}.${secret}`; } /** Parse an API key into its components; null if malformed. */ export function parseApiKey( key: string, -): { spaceSlug: string; lookupId: string; secret: string } | null { +): { lookupId: string; secret: string } | null { const parts = key.split("."); - if (parts.length !== 4) { + if (parts.length !== 3) { return null; } - const [prefix, spaceSlug, lookupId, secret] = parts; + const [prefix, lookupId, secret] = parts; if (prefix !== "me") return null; - if (!spaceSlug || !/^[a-z0-9]{12}$/.test(spaceSlug)) return null; if (!lookupId || !/^[A-Za-z0-9_-]{16}$/.test(lookupId)) return null; if (!secret || secret.length !== SECRET_LENGTH) return null; - return { spaceSlug, lookupId, secret }; + return { lookupId, secret }; +} + +/** + * True if the token is a legacy **space-scoped** api key + * (`me...`, the pre-global 4-part format). These no + * longer authenticate — callers use this to return a clear "recreate your key" + * error instead of a generic 401. New keys are 3-part (`parseApiKey`). + */ +export function isLegacyApiKey(token: string): boolean { + const parts = token.split("."); + return ( + parts.length === 4 && + parts[0] === "me" && + /^[a-z0-9]{12}$/.test(parts[1] ?? "") && + /^[A-Za-z0-9_-]{16}$/.test(parts[2] ?? "") && + (parts[3]?.length ?? 0) === SECRET_LENGTH + ); } diff --git a/packages/engine/core/db.integration.test.ts b/packages/engine/core/db.integration.test.ts index f32dfc3..ea2929c 100644 --- a/packages/engine/core/db.integration.test.ts +++ b/packages/engine/core/db.integration.test.ts @@ -126,14 +126,12 @@ test("createApiKey + validateApiKey (good / wrong secret)", async () => { }); test("api key string format round-trips with parseApiKey", async () => { - const slug = randomSlug(); const userId = await newUserId(); await db.createUser(userId, `erin_${userId.slice(0, 8)}`); const key = await db.createApiKey(userId, "fmt"); - const str = formatApiKey(slug, key.lookupId, key.secret); + const str = formatApiKey(key.lookupId, key.secret); expect(parseApiKey(str)).toEqual({ - spaceSlug: slug, lookupId: key.lookupId, secret: key.secret, }); diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts index 3f930c1..47956b5 100644 --- a/packages/engine/core/index.ts +++ b/packages/engine/core/index.ts @@ -3,6 +3,7 @@ export { generateLookupId, generateSecret, hashApiKeySecret, + isLegacyApiKey, parseApiKey, } from "./api-key"; export { type CoreStore, coreStore } from "./db"; diff --git a/packages/protocol/space/index.ts b/packages/protocol/space/index.ts index db52a1f..40dea9f 100644 --- a/packages/protocol/space/index.ts +++ b/packages/protocol/space/index.ts @@ -3,21 +3,11 @@ * POST /api/v1/memory/rpc alongside the memory.* data-plane methods. * * Follows the core model: principals (users/agents/groups), space membership, - * group membership, 3-level tree-access grants, and agent api keys. (Agent - * lifecycle is user-scoped and lives on the user endpoint; here agents are only - * referenced as members / api-key holders.) + * group membership, and 3-level tree-access grants. (Agent lifecycle and api + * keys are user-scoped and live on the user endpoint; here agents are only + * referenced as members.) */ import type { z } from "zod"; -import { - apiKeyCreateParams, - apiKeyCreateResult, - apiKeyDeleteParams, - apiKeyDeleteResult, - apiKeyGetParams, - apiKeyGetResult, - apiKeyListParams, - apiKeyListResult, -} from "./api-key.ts"; import { grantListParams, grantListResult, @@ -65,7 +55,6 @@ import { principalResolveResult, } from "./principal.ts"; -export * from "./api-key.ts"; export * from "./grant.ts"; export * from "./group.ts"; export * from "./invitation.ts"; @@ -79,7 +68,7 @@ function method( } /** - * Space management RPC method contract (member/agent/group/grant/apiKey). + * Space management RPC method contract (member/group/grant/invite). * Served on the memory endpoint together with the memory.* methods. */ export const spaceMethods = { @@ -115,12 +104,6 @@ export const spaceMethods = { "invite.create": method(inviteCreateParams, inviteCreateResult), "invite.list": method(inviteListParams, inviteListResult), "invite.revoke": method(inviteRevokeParams, inviteRevokeResult), - - // Api keys (4) - "apiKey.create": method(apiKeyCreateParams, apiKeyCreateResult), - "apiKey.list": method(apiKeyListParams, apiKeyListResult), - "apiKey.get": method(apiKeyGetParams, apiKeyGetResult), - "apiKey.delete": method(apiKeyDeleteParams, apiKeyDeleteResult), } as const; export type SpaceMethodName = keyof typeof spaceMethods; diff --git a/packages/protocol/space/api-key.ts b/packages/protocol/user/api-key.ts similarity index 84% rename from packages/protocol/space/api-key.ts rename to packages/protocol/user/api-key.ts index faaa372..5325ad8 100644 --- a/packages/protocol/space/api-key.ts +++ b/packages/protocol/user/api-key.ts @@ -1,9 +1,10 @@ /** * Api key method schemas (apiKey.*). * - * Keys are agent-only (humans authenticate via session). The plaintext key is - * returned exactly once, by apiKey.create. There is no soft-revoke state: - * apiKey.delete is the only removal (revoke ≡ delete). + * Keys are agent-only (humans authenticate via session) and global per-principal + * — not bound to a space. The plaintext key is returned exactly once, by + * apiKey.create. There is no soft-revoke state: apiKey.delete is the only + * removal (revoke ≡ delete). */ import { z } from "zod"; import { nameSchema, timestampSchema, uuidv7Schema } from "../fields.ts"; @@ -18,7 +19,7 @@ export const apiKeyInfoResponse = z.object({ }); export type ApiKeyInfoResponse = z.infer; -// apiKey.create — mint a key for an agent in this space +// apiKey.create — mint a key for an agent the caller owns export const apiKeyCreateParams = z.object({ agentId: uuidv7Schema, name: nameSchema, diff --git a/packages/protocol/user/index.ts b/packages/protocol/user/index.ts index cf15570..99d58da 100644 --- a/packages/protocol/user/index.ts +++ b/packages/protocol/user/index.ts @@ -1,7 +1,8 @@ /** * User RPC contract — session-only, user-scoped methods served on * POST /api/v1/user/rpc. Covers the lifecycle of a user's global service - * accounts (agents); space membership and api keys live on the space endpoint. + * accounts (agents) and their global api keys; space membership lives on the + * space endpoint. */ import type { z } from "zod"; @@ -15,6 +16,16 @@ import { agentRenameParams, agentRenameResult, } from "./agent.ts"; +import { + apiKeyCreateParams, + apiKeyCreateResult, + apiKeyDeleteParams, + apiKeyDeleteResult, + apiKeyGetParams, + apiKeyGetResult, + apiKeyListParams, + apiKeyListResult, +} from "./api-key.ts"; import { spaceCreateParams, spaceCreateResult, @@ -28,6 +39,7 @@ import { import { whoamiParams, whoamiResult } from "./whoami.ts"; export * from "./agent.ts"; +export * from "./api-key.ts"; export * from "./space.ts"; export * from "./whoami.ts"; @@ -38,7 +50,10 @@ function method( return { params, result }; } -/** User RPC method contract (identity + agent lifecycle + space discovery). */ +/** + * User RPC method contract (identity + agent lifecycle + api keys + space + * discovery). + */ export const userMethods = { whoami: method(whoamiParams, whoamiResult), @@ -47,6 +62,11 @@ export const userMethods = { "agent.rename": method(agentRenameParams, agentRenameResult), "agent.delete": method(agentDeleteParams, agentDeleteResult), + "apiKey.create": method(apiKeyCreateParams, apiKeyCreateResult), + "apiKey.list": method(apiKeyListParams, apiKeyListResult), + "apiKey.get": method(apiKeyGetParams, apiKeyGetResult), + "apiKey.delete": method(apiKeyDeleteParams, apiKeyDeleteResult), + "space.list": method(spaceListParams, spaceListResult), "space.create": method(spaceCreateParams, spaceCreateResult), "space.rename": method(spaceRenameParams, spaceRenameResult), diff --git a/packages/server/middleware/authenticate-space.integration.test.ts b/packages/server/middleware/authenticate-space.integration.test.ts index 1d25faa..4fb94f9 100644 --- a/packages/server/middleware/authenticate-space.integration.test.ts +++ b/packages/server/middleware/authenticate-space.integration.test.ts @@ -10,12 +10,14 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; import { authStore } from "@memory.build/auth"; import { bootstrapSpaceDatabase, + generateSlug, migrateAuth, migrateCore, + provisionSpace, } from "@memory.build/database"; import * as engineCore from "@memory.build/engine/core"; import postgres, { type Sql } from "postgres"; -import { provisionUser } from "../provision"; +import { addSpaceCreator, provisionUser } from "../provision"; import { authenticateSpace, SPACE_HEADER } from "./authenticate-space"; const URL = @@ -125,11 +127,7 @@ test("api key: agent of the space resolves with apiKeyId set", async () => { engineCore.ACCESS.read, ); const key = await core.createApiKey(agentId, "ci"); - const fullKey = engineCore.formatApiKey( - p.spaceSlug, - key.lookupId, - key.secret, - ); + const fullKey = engineCore.formatApiKey(key.lookupId, key.secret); const result = await authenticateSpace( req({ token: fullKey, space: p.spaceSlug }), @@ -143,6 +141,55 @@ test("api key: agent of the space resolves with apiKeyId set", async () => { } }); +test("api key is global: one key authenticates into every space the agent belongs to", async () => { + const p = await provision(); + const core = engineCore.coreStore(sql, coreSchema); + + // A second space also created by p, so p (the agent's owner) has access in + // both — the agent's effective access is clamped to its owner's. + const slug2 = generateSlug(); + const spaceId2 = await core.createSpace(slug2, "second"); + await provisionSpace(sql, { slug: slug2 }); + createdSpaceSchemas.push(`me_${slug2}`); + await addSpaceCreator(core, spaceId2, p.userId); + + const agentId = await core.createAgent(p.userId, `agent-${rand()}`); + for (const sid of [p.spaceId, spaceId2]) { + await core.addPrincipalToSpace(sid, agentId); + await core.grantTreeAccess(sid, agentId, "share", engineCore.ACCESS.read); + } + const key = await core.createApiKey(agentId, "ci"); + const fullKey = engineCore.formatApiKey(key.lookupId, key.secret); + + for (const slug of [p.spaceSlug, slug2]) { + const result = await authenticateSpace( + req({ token: fullKey, space: slug }), + deps(), + ); + expect(result.ok).toBe(true); + if (result.ok) expect(result.context.principalId).toBe(agentId); + } +}); + +test("legacy 4-part api key → 401 with a LEGACY_API_KEY recreate message", async () => { + const p = await provision(); + // A token shaped like the retired me... format. + const legacy = `me.${p.spaceSlug}.${"a".repeat(16)}.${"s".repeat(32)}`; + const result = await authenticateSpace( + req({ token: legacy, space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.status).toBe(401); + const body = (await result.error.json()) as { + error: { code: string; message: string }; + }; + expect(body.error.code).toBe("LEGACY_API_KEY"); + expect(body.error.message).toContain("me apikey create"); + } +}); + test("missing Authorization → 401", async () => { const result = await authenticateSpace( req({ space: "abcdef012345" }), @@ -179,30 +226,28 @@ test("invalid session token → 401", async () => { if (!result.ok) expect(result.error.status).toBe(401); }); -test("api key slug ≠ header → 400 (SPACE_MISMATCH)", async () => { +test("api key: agent with no access in the requested space → 403", async () => { const p = await provision(); const other = await provision(); const core = engineCore.coreStore(sql, coreSchema); const agentId = await core.createAgent(p.userId, `agent-${rand()}`); + await core.addPrincipalToSpace(p.spaceId, agentId); await core.grantTreeAccess( p.spaceId, agentId, - engineCore.ROOT_PATH, + "share", engineCore.ACCESS.read, ); const key = await core.createApiKey(agentId, "ci"); - // key minted for p's slug, but header points at a different space - const fullKey = engineCore.formatApiKey( - p.spaceSlug, - key.lookupId, - key.secret, - ); + // A valid global key, but the agent has no access in `other` — the access gate + // (build_tree_access empty) denies it rather than a parse-time rejection. + const fullKey = engineCore.formatApiKey(key.lookupId, key.secret); const result = await authenticateSpace( req({ token: fullKey, space: other.spaceSlug }), deps(), ); expect(result.ok).toBe(false); - if (!result.ok) expect(result.error.status).toBe(400); + if (!result.ok) expect(result.error.status).toBe(403); }); test("session: member of another space has no grant here → 403", async () => { diff --git a/packages/server/middleware/authenticate-space.ts b/packages/server/middleware/authenticate-space.ts index e68b55a..05291f8 100644 --- a/packages/server/middleware/authenticate-space.ts +++ b/packages/server/middleware/authenticate-space.ts @@ -5,18 +5,20 @@ * (`treeAccess`) that the space SQL functions consume. Two credential modes, * discriminated by whether the bearer token parses as an api key: * - * - api key (agent): `me...` — validated against core. + * - api key (agent): `me..` — validated against core. * - session (human): an opaque session token — validated against auth. * * The space is always selected by the `X-Me-Space` header (uniform for both * modes). `core.buildTreeAccess(principalId, space.id)` is the single - * authorization gate: a principal with no grants in the space (including an api - * key minted for a different space) resolves to an empty set and is denied. + * authorization gate: a principal with no grants in the space resolves to an + * empty set and is denied. Api keys are global, so a key whose principal isn't a + * member of the requested space is denied here rather than at parse time. */ import type { AuthStore } from "@memory.build/auth"; import { slugToSchema } from "@memory.build/database"; import { type CoreStore, + isLegacyApiKey, parseApiKey, type Space, type TreeAccess, @@ -116,22 +118,9 @@ async function authenticateSpaceInner( let apiKeyId: string | null; if (parsed) { - // The api key embeds its own slug; assert it matches the header so a - // misrouted key gives a clear error rather than a confusing 403 below. - if (parsed.spaceSlug !== slug) { - debug("space auth failed: api key slug != header", { - slug, - keySlug: parsed.spaceSlug, - }); - return { - ok: false, - error: error( - `API key is not valid for space ${slug}`, - 400, - "SPACE_MISMATCH", - ), - }; - } + // Api keys are global; the space comes solely from the header. A key whose + // principal isn't a member of this space falls through to the empty-access + // gate below (403), not a parse-time rejection. const validated = await core.validateApiKey(parsed.lookupId, parsed.secret); if (!validated) { debug("space auth failed: invalid api key"); @@ -139,6 +128,19 @@ async function authenticateSpaceInner( } principalId = validated.memberId; apiKeyId = validated.apiKeyId; + } else if (isLegacyApiKey(token)) { + // A pre-global 4-part key (me...). These no longer + // authenticate; tell the operator to recreate the key rather than failing + // with a confusing generic 401. + debug("space auth failed: legacy 4-part api key"); + return { + ok: false, + error: error( + "This API key uses the old space-scoped format (me...) and no longer works. Recreate it with `me apikey create `, then update ME_API_KEY or your MCP/plugin config.", + 401, + "LEGACY_API_KEY", + ), + }; } else { const session = await auth.validateSession(token); if (!session) { diff --git a/packages/server/rpc/memory/index.ts b/packages/server/rpc/memory/index.ts index 95df6f4..1b6bef4 100644 --- a/packages/server/rpc/memory/index.ts +++ b/packages/server/rpc/memory/index.ts @@ -3,10 +3,9 @@ * * The new-model replacement for the engine RPC, combining the memory data-plane * methods (spaceStore) with the space management methods (coreStore): membership, - * agents, groups, tree-access grants, and agent api keys. + * groups, tree-access grants, and invitations. */ import type { MethodRegistry } from "../types"; -import { apiKeyMethods } from "./api-key"; import { grantMethods } from "./grant"; import { groupMethods } from "./group"; import { invitationMethods } from "./invitation"; @@ -21,7 +20,7 @@ export { /** * The full memory-endpoint registry: data-plane + space management methods. - * (Agent lifecycle lives on the user endpoint — see rpc/user.) + * (Agent lifecycle and api keys live on the user endpoint — see rpc/user.) */ export const memoryMethods: MethodRegistry = new Map([ ...memoryDataMethods, @@ -29,5 +28,4 @@ export const memoryMethods: MethodRegistry = new Map([ ...groupMethods, ...grantMethods, ...invitationMethods, - ...apiKeyMethods, ]); diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index 2b64c7f..71f84be 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -1,7 +1,8 @@ // Integration test for the space management handlers (4C-2b): member / agent / -// group / grant / apiKey, driven through the merged memory registry against a +// group / grant / invite, driven through the merged memory registry against a // provisioned space. The provisioned owner has owner@root, satisfying the -// management authorization gate. +// management authorization gate. (Api keys are user-endpoint — see +// rpc/user/api-key.integration.test.ts.) // TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ // bun test --timeout 30000 \ // packages/server/rpc/memory/management.integration.test.ts @@ -367,44 +368,6 @@ test("invite.* require space-admin authority (owner@root is not enough)", async ); }); -test("apiKey: create (agent-only) / list / get / delete", async () => { - // agent lifecycle is the user endpoint's job; here the owner brings an agent - // into the space, then mints its (space-bound) key. - const agentId = await makeAgent(ownerId); - await call("principal.add", { principalId: agentId }); - const created = await call<{ id: string; key: string }>("apiKey.create", { - agentId, - name: "ci", - }); - expect(created.key.startsWith(`me.${space.slug}.`)).toBe(true); - - const list = await call<{ apiKeys: { id: string }[] }>("apiKey.list", { - memberId: agentId, - }); - expect(list.apiKeys.map((k) => k.id)).toContain(created.id); - - const got = await call<{ apiKey: { id: string } | null }>("apiKey.get", { - id: created.id, - }); - expect(got.apiKey?.id).toBe(created.id); - - expect( - (await call<{ deleted: boolean }>("apiKey.delete", { id: created.id })) - .deleted, - ).toBe(true); - expect( - (await call<{ apiKey: unknown }>("apiKey.get", { id: created.id })).apiKey, - ).toBeNull(); -}); - -test("apiKey.create rejects a non-agent member", async () => { - // ownerId is a user, not an agent → NOT_FOUND in this space's agents - await expectAppError( - call("apiKey.create", { agentId: ownerId, name: "nope" }), - "NOT_FOUND", - ); -}); - test("roster/group management requires admin or owner", async () => { // a plain member: write access on a subtree, not an admin, not a root owner const member = await makeUser(); @@ -655,13 +618,8 @@ test("self-service: a non-owner member brings their own agent into the space", a ).added, ).toBe(true); - // and mint its key + self-grant it (capped by their own access) - const key = await call<{ key: string }>( - "apiKey.create", - { agentId, name: "k" }, - as, - ); - expect(key.key.startsWith(`me.${space.slug}.`)).toBe(true); + // and self-grant it (capped by their own access). Minting the agent's api key + // is a user-endpoint op (apiKey.* — see rpc/user/api-key.integration.test.ts). expect( ( await call<{ granted: boolean }>( diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts index 042b9a4..8fb5e5a 100644 --- a/packages/server/rpc/memory/support.ts +++ b/packages/server/rpc/memory/support.ts @@ -1,6 +1,6 @@ /** - * Shared helpers for the space management handlers (member/agent/group/grant/ - * apiKey): the owner authorization gate, core SQL error mapping, and response + * Shared helpers for the space management handlers (member/group/grant/invite): + * the owner authorization gate, core SQL error mapping, and response * serializers. */ @@ -11,7 +11,6 @@ import { TreePathError, } from "@memory.build/database"; import type { - ApiKeyInfo, Group, GroupMember, GroupMembership, @@ -21,7 +20,6 @@ import type { } from "@memory.build/engine/core"; import { ACCESS, ROOT_PATH } from "@memory.build/engine/core"; import type { - ApiKeyInfoResponse, GroupMemberResponse, GroupMembershipResponse, GroupResponse, @@ -159,27 +157,6 @@ export async function callerOwnsAgent( return agent !== undefined && agent.ownerId === context.principalId; } -/** - * Assert the caller owns `agentId` (an agent in this space). NOT_FOUND if the - * agent isn't a member of this space; FORBIDDEN if it's owned by someone else. - */ -export async function requireOwnedAgent( - context: SpaceRpcContext, - agentId: string, -): Promise { - const agents = await context.core.listSpacePrincipals(context.space.id, "a"); - const agent = agents.find((a) => a.id === agentId); - if (!agent) { - throw new AppError( - "NOT_FOUND", - `Agent not found in this space: ${agentId}`, - ); - } - if (agent.ownerId !== context.principalId) { - throw new AppError("FORBIDDEN", "Not the owner of this agent"); - } -} - /** * True if `principalId` is an agent owned by the caller, checked globally (not * scoped to the current space). Used by principal.add so a member can bring @@ -259,17 +236,6 @@ export function toTreeGrantResponse( }; } -export function toApiKeyInfoResponse(k: ApiKeyInfo): ApiKeyInfoResponse { - return { - id: k.id, - memberId: k.memberId, - lookupId: k.lookupId, - name: k.name, - createdAt: k.createdAt.toISOString(), - expiresAt: k.expiresAt?.toISOString() ?? null, - }; -} - export function toSpaceInvitationResponse( i: SpaceInvitation, ): SpaceInvitationResponse { diff --git a/packages/server/rpc/user/agent.ts b/packages/server/rpc/user/agent.ts index de5661a..406d876 100644 --- a/packages/server/rpc/user/agent.ts +++ b/packages/server/rpc/user/agent.ts @@ -2,9 +2,9 @@ * Agent handlers (agent.*) for the user RPC. * * Agents are a user's global service accounts. The lifecycle here is purely - * user-scoped: create / list / rename / delete the caller's own agents. - * Bringing an agent into a space (principal.add) and minting its space-bound - * api key (apiKey.create) are space-endpoint operations. + * user-scoped: create / list / rename / delete the caller's own agents, and + * mint their (global) api keys (apiKey.* — see ./api-key.ts). Bringing an agent + * into a space (principal.add) is a space-endpoint operation. */ import type { Principal } from "@memory.build/engine/core"; import type { @@ -40,7 +40,7 @@ function toAgentResponse(p: Principal): AgentResponse { } /** Assert the caller owns this agent (globally). */ -async function requireOwnAgent( +export async function requireOwnAgent( ctx: UserRpcContext, agentId: string, ): Promise { diff --git a/packages/server/rpc/user/api-key.integration.test.ts b/packages/server/rpc/user/api-key.integration.test.ts new file mode 100644 index 0000000..a192874 --- /dev/null +++ b/packages/server/rpc/user/api-key.integration.test.ts @@ -0,0 +1,154 @@ +// Integration test for the user RPC api-key handlers (apiKey.* lifecycle). +// Keys are agent-only and global: minting one needs only agent ownership — no +// space membership — and the key string carries no space slug. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/user/api-key.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import { coreStore } from "@memory.build/engine/core"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import type { HandlerContext } from "../types"; +import { userMethods } from "./index"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let coreSchema: string; +let authSchema: string; +let userId: string; + +function call( + method: string, + params: unknown, + asUser: string = userId, +): Promise { + const registered = userMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/user/rpc"), + core: coreStore(sql, coreSchema), + auth: authStore(sql, authSchema), + userId: asUser, + db: sql, + coreSchema, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +async function makeUser(): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await coreStore(sql, coreSchema).createUser(id, `u_${rand(8)}@example.com`); + return id; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + authSchema = `auth_test_${rand(8)}`; + await bootstrapSpaceDatabase(sql); + await migrateCore(sql, { schema: coreSchema }); + await migrateAuth(sql, { schema: authSchema }); +}); + +afterAll(async () => { + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + userId = await makeUser(); +}); + +test("create (global, no space needed) / list / get / delete", async () => { + // The agent is owned by the caller but is NOT a member of any space — key + // creation depends only on ownership, not space membership. + const { id: agentId } = await call<{ id: string }>("agent.create", { + name: "bot", + }); + + const created = await call<{ id: string; key: string }>("apiKey.create", { + agentId, + name: "ci", + expiresAt: null, + }); + // Global format: me.. — no space slug. + expect(created.key).toMatch(/^me\.[A-Za-z0-9_-]{16}\.[A-Za-z0-9_-]{32}$/); + + const list = await call<{ apiKeys: { id: string }[] }>("apiKey.list", { + memberId: agentId, + }); + expect(list.apiKeys.map((k) => k.id)).toContain(created.id); + + const got = await call<{ apiKey: { id: string } | null }>("apiKey.get", { + id: created.id, + }); + expect(got.apiKey?.id).toBe(created.id); + + expect( + (await call<{ deleted: boolean }>("apiKey.delete", { id: created.id })) + .deleted, + ).toBe(true); + expect( + (await call<{ apiKey: unknown }>("apiKey.get", { id: created.id })).apiKey, + ).toBeNull(); +}); + +test("apiKey.create rejects a non-agent member", async () => { + // a user id is not an agent → NOT_FOUND + await expectAppError( + call("apiKey.create", { agentId: userId, name: "nope", expiresAt: null }), + "NOT_FOUND", + ); +}); + +test("cannot manage keys for another user's agent", async () => { + const { id: agentId } = await call<{ id: string }>("agent.create", { + name: "mine", + }); + const intruder = await makeUser(); + await expectAppError( + call("apiKey.create", { agentId, name: "x", expiresAt: null }, intruder), + "FORBIDDEN", + ); + await expectAppError( + call("apiKey.list", { memberId: agentId }, intruder), + "FORBIDDEN", + ); +}); + +test("apiKey.get is null for an unknown key id", async () => { + const [row] = await sql`select uuidv7() as id`; + const got = await call<{ apiKey: unknown }>("apiKey.get", { + id: row?.id as string, + }); + expect(got.apiKey).toBeNull(); +}); diff --git a/packages/server/rpc/memory/api-key.ts b/packages/server/rpc/user/api-key.ts similarity index 55% rename from packages/server/rpc/memory/api-key.ts rename to packages/server/rpc/user/api-key.ts index af93e31..83c4f0f 100644 --- a/packages/server/rpc/memory/api-key.ts +++ b/packages/server/rpc/user/api-key.ts @@ -1,8 +1,12 @@ /** - * Api key handlers (apiKey.*). Keys are agent-only and self-service: the caller - * manages keys for agents they own. The plaintext key is returned once by + * Api key handlers (apiKey.*) for the user RPC. + * + * Keys are agent-only and self-service: the caller manages keys for agents they + * own. Keys are global per-principal (not space-bound) — the same key works in + * any space the agent is admitted to. The plaintext key is returned once by * create. Revoke ≡ delete (no soft-revoke state). */ +import type { ApiKeyInfo } from "@memory.build/engine/core"; import { formatApiKey } from "@memory.build/engine/core"; import type { ApiKeyCreateParams, @@ -11,36 +15,49 @@ import type { ApiKeyDeleteResult, ApiKeyGetParams, ApiKeyGetResult, + ApiKeyInfoResponse, ApiKeyListParams, ApiKeyListResult, -} from "@memory.build/protocol/space"; +} from "@memory.build/protocol/user"; import { apiKeyCreateParams, apiKeyDeleteParams, apiKeyGetParams, apiKeyListParams, -} from "@memory.build/protocol/space"; +} from "@memory.build/protocol/user"; +import { guardCore } from "../core-error"; import { buildRegistry } from "../registry"; import type { HandlerContext } from "../types"; -import { guardCore, requireOwnedAgent, toApiKeyInfoResponse } from "./support"; -import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; +import { requireOwnAgent } from "./agent"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +function toApiKeyInfoResponse(k: ApiKeyInfo): ApiKeyInfoResponse { + return { + id: k.id, + memberId: k.memberId, + lookupId: k.lookupId, + name: k.name, + createdAt: k.createdAt.toISOString(), + expiresAt: k.expiresAt?.toISOString() ?? null, + }; +} async function apiKeyCreate( params: ApiKeyCreateParams, context: HandlerContext, ): Promise { - assertSpaceRpcContext(context); - const ctx = context as SpaceRpcContext; - // Keys are agent-only; the caller must own the agent (which is in this space). - await requireOwnedAgent(ctx, params.agentId); + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + // Keys are agent-only; the caller must own the agent (checked globally). + await requireOwnAgent(ctx, params.agentId); const created = await guardCore(() => ctx.core.createApiKey(params.agentId, params.name, { expiresAt: params.expiresAt ? new Date(params.expiresAt) : undefined, }), ); - // The full key string embeds the space slug for routing; returned once. - const key = formatApiKey(ctx.space.slug, created.lookupId, created.secret); + // The full key string is global (no space slug); returned once. + const key = formatApiKey(created.lookupId, created.secret); return { id: created.id, key }; } @@ -48,9 +65,9 @@ async function apiKeyList( params: ApiKeyListParams, context: HandlerContext, ): Promise { - assertSpaceRpcContext(context); - const ctx = context as SpaceRpcContext; - await requireOwnedAgent(ctx, params.memberId); + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireOwnAgent(ctx, params.memberId); const keys = await ctx.core.listApiKeys(params.memberId); return { apiKeys: keys.map(toApiKeyInfoResponse) }; } @@ -59,12 +76,12 @@ async function apiKeyGet( params: ApiKeyGetParams, context: HandlerContext, ): Promise { - assertSpaceRpcContext(context); - const ctx = context as SpaceRpcContext; + assertUserRpcContext(context); + const ctx = context as UserRpcContext; const key = await ctx.core.getApiKey(params.id); if (!key) return { apiKey: null }; // Only the owning user of the key's agent may see it. - await requireOwnedAgent(ctx, key.memberId); + await requireOwnAgent(ctx, key.memberId); return { apiKey: toApiKeyInfoResponse(key) }; } @@ -72,11 +89,11 @@ async function apiKeyDelete( params: ApiKeyDeleteParams, context: HandlerContext, ): Promise { - assertSpaceRpcContext(context); - const ctx = context as SpaceRpcContext; + assertUserRpcContext(context); + const ctx = context as UserRpcContext; const key = await ctx.core.getApiKey(params.id); if (!key) return { deleted: false }; - await requireOwnedAgent(ctx, key.memberId); + await requireOwnAgent(ctx, key.memberId); const deleted = await guardCore(() => ctx.core.deleteApiKey(params.id)); return { deleted }; } diff --git a/packages/server/rpc/user/index.ts b/packages/server/rpc/user/index.ts index f7721e6..87e60b9 100644 --- a/packages/server/rpc/user/index.ts +++ b/packages/server/rpc/user/index.ts @@ -1,9 +1,10 @@ /** * User RPC method registry — served at `/api/v1/user/rpc` (session-only, - * user-scoped). Currently the lifecycle of a user's agents. + * user-scoped): the lifecycle of a user's agents and their global api keys. */ import type { MethodRegistry } from "../types"; import { agentMethods } from "./agent"; +import { apiKeyMethods } from "./api-key"; import { spaceMethods } from "./space"; import { whoamiMethods } from "./whoami"; @@ -13,9 +14,13 @@ export { type UserRpcContext, } from "./types"; -/** The user-endpoint registry: identity + agent lifecycle + space discovery. */ +/** + * The user-endpoint registry: identity + agent lifecycle + api keys + space + * discovery. + */ export const userMethods: MethodRegistry = new Map([ ...whoamiMethods, ...agentMethods, + ...apiKeyMethods, ...spaceMethods, ]); From 3dfd67ce8ae7f60f49f9ce85381032073fb997ed Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 15:02:25 +0200 Subject: [PATCH 094/156] docs: restructure REDESIGN_DIFFERENCES TL;DR + record the access-fn decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split §1 into "Differences" (A/B/D/E) and "Not implemented" (C/F/G/H), add a Decision column, and adjudicate item D (access function): keep the current build_tree_access(_member_id, _space_id) -> jsonb over the redesign's effective_tree_access(...) -> table, with per-axis reasoning in §3.D. Move the per-space embedding item (C) into the not-implemented section (§4). Co-Authored-By: Claude Opus 4.8 (1M context) --- REDESIGN_DIFFERENCES.md | 127 ++++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index c4bd8e5..81a5def 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -15,18 +15,31 @@ File references are `path:line` against the repo root. --- -## 1. TL;DR — the substantive divergences - -| # | Topic | Redesign says | Implementation does | Severity | -|---|-------|---------------|---------------------|----------| -| A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | -| B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | -| C | Embedding config | Per-space model/dimension, recorded in `core.space` | Hardcoded `text-embedding-3-small` / 1536 for all spaces; `core.space` has a TODO comment, no such columns | **Major** | -| D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | -| E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | -| F | Last-admin safeguard | Must prevent removing/demoting the last admin | **Not implemented** | Gap | -| G | `me memory copy`/`cp` | Listed | **Not implemented** | Gap | -| H | `me user group list` | The one supported user command for v1 | **Not implemented** | Gap | +## 1. TL;DR + +### 1a. Differences + +Where the doc and the code diverge but both build the feature. The **Decision** +column records items we've adjudicated (which side to keep); `—` = not yet +decided. Items the redesign lists but the code does **not** build live in §1b. + +| # | Topic | Redesign says | Implementation does | Severity | Decision | +|---|-------|---------------|---------------------|----------|----------| +| A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | — | +| B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | — | +| D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | **Keep current** (see §3.D) | +| E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | — | + +### 1b. Not implemented (gaps vs. the redesign) + +Listed in the redesign but not built. Detail in §4. + +| # | Topic | Redesign wants | Status | +|---|-------|----------------|--------| +| C | Embedding config | Per-space model/dimension, recorded in `core.space` | Not implemented (hardcoded uniform; templated DDL only) | +| F | Last-admin safeguard | Must prevent removing/demoting the last admin | Not implemented | +| G | `me memory copy`/`cp` | Listed in §"Memory Commands" | Not implemented | +| H | `me user group list` | The one supported `me user` command for v1 | Not implemented | The agent access-masking model (the part the doc was least confident about) **is** implemented as designed — see §6. @@ -84,36 +97,11 @@ respected. But the **convention layer the redesign deferred is shipped**, and th creator's grant is `home`+`share` rather than `root`. Any reader of REDESIGN.md would expect a fresh space to be empty and root-owned; it is neither. -### C. Embedding model/dimension is hardcoded, not per-space - -The redesign (§"Space") wants the embedding model and dimension to be per-space, -templated into the DDL, and recorded in `core.space` so the server can route -embedding work. - -Reality is split: - -- The DDL **is** templated: `embedding halfvec({{embedding_dimensions}})` - (`space/migrate/incremental/001_memory.sql:10`). So the *mechanism* exists. -- But the value is **hardcoded to 1536 / `text-embedding-3-small` for every - space** server-side (`packages/server/config.ts:8`, `packages/server/index.ts:212` - comment: "Model and dimensions are hardcoded - all spaces use the same - embedding model"). -- `core.space` does **not** record provider/model/dimension. It literally carries - a TODO: `-- we likely need columns for embedding provider, model, dimensions` - (`core/migrate/incremental/001_space.sql:9`). There is also **no shard / - placement column**, though the redesign (§`core.space`) says the space record - "tracks placement information, such as the shard." - -So the "per-space embedding" and "placement metadata in `core.space`" parts of the -redesign are not realized; all spaces are uniform and single-DB (consistent with -the "no sharding in v1" non-goal, but the metadata hooks the redesign called for -aren't there yet). - --- ## 3. Naming / shape divergences (same concept, different surface) -### D. Access resolution function +### D. Access resolution function — decision: **keep the current implementation** - Redesign: `core.effective_tree_access(_space_id uuid, _principal_id uuid) returns table(tree_path ltree, access int4)`. @@ -122,11 +110,46 @@ aren't there yet). **argument order**, takes **`_member_id`** (not `principal_id`), and returns a **JSONB array** of `{tree_path, access}` objects rather than a SQL table. -Space functions consume it as a `_tree_access jsonb` argument -(`space/migrate/idempotent/001_memory.sql:33`), matching the redesign's -"pushed-down JSONB access set" future shape — but that's the *only* code path, -not a later optimization. Per CLAUDE.md the auth gate is "non-empty -`build_tree_access`." +After evaluating the two, the current implementation is **better or equal on +every axis** — so we keep the code and (per §7) treat REDESIGN.md's signature as +the weaker spec to be updated. Reasoning per axis: + +- **Parameter `_member_id` (current) > `_principal_id` (redesign) — correctness.** + Effective access is only ever computed for an *authenticating actor* (a user or + agent). A group never authenticates and isn't owner-maskable, and + `build_tree_access` only dispatches `'u'`/`'a'`. `member_id` (the u|a-only + generated column) encodes that constraint in the signature; the redesign's + `_principal_id` is looser and wrongly implies a group could be passed. +- **Argument order `(member_id, space_id)` (current) — consistency.** The whole + helper family is subject-first: `user_tree_access(_user_id, _space_id)`, + `agent_tree_access(_agent_id, _space_id)`, `member_tree_access(…, _space_id)` + (`003_tree_access.sql`). The redesign's space-first order would make the public + entry the lone exception. +- **Return type `jsonb` (current) vs `table` (redesign) — `jsonb` fits this + architecture; the table form's edge is unrealized.** The set always + round-trips through the application layer: `build_tree_access` → TS array + (`packages/engine/core/db.ts:429`), where the app uses it for the **auth gate** + (`treeAccess.length === 0` → 403, `authenticate-space.ts`) and **owner checks** + (`rpc/memory/support.ts`), then passes it **back into every space function as a + jsonb argument** (`sql.json(treeAccess)::jsonb`, + `packages/engine/space/db.ts:101`; consumed via `jsonb_to_recordset` in + `space/migrate/idempotent/001_memory.sql:33`). The only advantage of a + table-returning function — joining it directly in SQL — is never used, because + nothing computes access purely in-SQL; the app is always in the loop. And it + would save nothing: postgres.js parses either return shape into the same + `[{tree_path, access}]` JS array, and the app re-serializes with `sql.json(…)` + on the way back down regardless. Meanwhile `jsonb` matches the future sharded + pushdown with **zero refactor**, and the typed/composable layer the redesign + wanted **already exists internally** as the `*_tree_access` table functions — + `build_tree_access` is explicitly just the jsonb *bridge* on top of them. +- **Name `build_` vs `effective_` — cosmetic, the only place the redesign reads + nicer.** "Effective access" is the precise term for net/resolved permissions; + "build" describes the bridge role (its doc comment: "the bridge … returns … + the jsonb array shape"). Not worth a rename across SQL + `CLAUDE.md` + TS. If a + more semantic public name is ever wanted, `effective_tree_access` (returning + jsonb) is the one thing worth lifting from the redesign. + +Per CLAUDE.md the auth gate is "non-empty `build_tree_access`." ### E. Two RPC endpoints, plus REST auth @@ -159,6 +182,18 @@ All same-intent, different spelling: ## 4. Not yet implemented (gaps vs. the redesign) +- **C. Per-space embedding config + placement metadata.** The redesign (§"Space") + wants the embedding model/dimension per-space, templated into the DDL, and + recorded in `core.space`; the space record should also track placement (the + shard). Built only partway: the DDL **is** templated + (`embedding halfvec({{embedding_dimensions}})`, + `space/migrate/incremental/001_memory.sql:10`), but the value is **hardcoded to + 1536 / `text-embedding-3-small` for every space** (`packages/server/config.ts:8`, + `packages/server/index.ts:212`). `core.space` records neither model/dimension + (it carries a TODO: `-- we likely need columns for embedding provider, model, + dimensions`, `core/migrate/incremental/001_space.sql:9`) nor a shard/placement + column. Consistent with the "no sharding in v1" non-goal, but the per-space + hooks the redesign called for aren't there yet. - **F. Last-admin safeguard.** The redesign requires that no cascade or removal may strip the last `principal_space.admin = true` from a space (§"Last-Admin Safeguard", §`core.principal_space`). No such check exists in SQL or app code. @@ -269,9 +304,11 @@ If REDESIGN.md is meant to track reality, these lines are now stale: - §"Space": `core.space` does not yet carry embedding or placement columns (there's a TODO); embedding is hardcoded uniform. Either implement or downgrade the prose to "future." -- §"Authorization Boundary": rename `effective_tree_access(_space_id, +- §"Authorization Boundary": update `effective_tree_access(_space_id, _principal_id) returns table` to the real `build_tree_access(_member_id, - _space_id) returns jsonb`. + _space_id) returns jsonb` — the current signature is the preferred one (see + §3.D for the per-axis reasoning); optionally adopt the name `effective_…` while + keeping the `_member_id` argument and jsonb return. - Command list: `space alter`→`rename`, `apikey revoke`→`delete`, `agent group list`→`agent groups`; add `me agent add`; `me memory copy` and `me user group list` are unbuilt; the single-API framing should mention the two RPC endpoints From 3497440872ac141b6043de7a01fc657e42adb46f Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 16:14:37 +0200 Subject: [PATCH 095/156] docs: record auth-schema decision (keep separate; no cross-schema FK) Adjudicate item A in REDESIGN_DIFFERENCES: keep the separate `auth` schema over the redesign's auth-in-core, with per-axis reasoning (separation of concerns, established auth shape, the shared-id pattern, future optionality) and caveats. Record the related "no cross-schema FK between core.principal and auth.users" call in DECISIONS_FOR_REVIEW (defer; revisit with user-deletion / standalone users). Co-Authored-By: Claude Opus 4.8 (1M context) --- DECISIONS_FOR_REVIEW.md | 50 +++++++++++++++++++++++++++++++++++++++++ REDESIGN_DIFFERENCES.md | 44 +++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index 6fa9a05..d547b4f 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -121,3 +121,53 @@ stays interactive/session-only — an api key is a long-lived bearer secret, so making them mintable for users widens that surface. **Status:** needs decision. + +--- + +## No cross-schema FK between `core.principal` and `auth.users` + +**Date:** 2026-06-06 · **Area:** auth / core schema boundary + +For a user principal, `auth.users.id == core.principal.id`. That invariant is +**app-enforced only** — `provisionUser` writes both rows with the same id in one +`sql.begin` transaction (`packages/server/provision.ts:80,89`), and the two +schemas reference each other nowhere (`core.principal` has no FK to `auth.users`; +the `auth` migrations never mention `core`). **Decision: keep it app-enforced — +do not add a DB-level cross-schema FK now.** + +**Alternative considered:** add `core.principal.user_id references auth.users(id) +on delete cascade`. This is clean in shape — `user_id` is the generated column +(`= id` when `kind='u'`, else null) and FKs ignore null columns, so it would +constrain *only* user principals and leave agents/groups untouched; the cascade +would also make "delete an identity" tear down the principal + its grant graph in +one statement. + +**Why defer:** + +- **It makes migration order load-bearing, and today it isn't.** `auth` and + `core` are independent migrate runners; call sites order them inconsistently + (`authenticate-space` migrates auth→core; the agent/api-key integration tests + migrate core→auth). A core→auth FK forces auth-before-core everywhere and would + require standardizing production orchestration + fixing those test setups. +- **It forecloses the deliberate split-DB hedge.** The no-FK decoupling is + intentional — `packages/database/index.ts` notes `auth` could be "distributed + across databases again" (it *was* a separate DB before the recent + consolidation). A cross-schema FK only works within one database. +- **The drift it guards against is near-zero today.** The invariant has exactly + one writer (`provisionUser`, atomic), there's no user-deletion flow yet, and in + v1 every user principal is created via OAuth login (so always has an + `auth.users` row). +- **It would prematurely settle a deferred design question** — "standalone + non-OAuth users" (service accounts) are deferred; a hard FK bakes in "every user + principal has an `auth.users` row," which should be decided when that lands. + +**How to change it (add the FK):** add `core.principal.user_id references +auth.users(id) on delete cascade` (uses the existing u-only generated column), +standardize the migration order to **auth-first** (production + the integration +test `beforeAll`s), and decide whether standalone users get an `auth.users` +identity row. The natural moment is when adding a `user delete` flow or finalizing +standalone users — the cascade-on-identity-delete becomes a concrete win then. A +cheap interim guard: a test asserting every `core.principal` `kind='u'` has a +matching `auth.users` and vice versa. + +**Status:** decided (defer); revisit with user-deletion / standalone users. diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index 81a5def..4e7b0a8 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -25,7 +25,7 @@ decided. Items the redesign lists but the code does **not** build live in §1b. | # | Topic | Redesign says | Implementation does | Severity | Decision | |---|-------|---------------|---------------------|----------|----------| -| A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | — | +| A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | **Keep current** (see §2.A) | | B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | — | | D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | **Keep current** (see §3.D) | | E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | — | @@ -48,7 +48,7 @@ implemented as designed — see §6. ## 2. Architectural divergences -### A. Auth lives in a separate `auth` schema, not in `core` +### A. Auth lives in a separate `auth` schema, not in `core` — decision: **keep the current implementation** The redesign places all authentication state in `core`: `core.session`, `core.oauth_identity`, `core.oauth_flow`. The implementation instead uses a @@ -68,6 +68,43 @@ redesign (§"Authorization Boundary") is true for *authorization* (principals, grants, groups) but **not** for *authentication* — authentication is its own schema. The redesign never mentions an `auth` schema or better-auth. +After evaluating the two, **keep the separate `auth` schema** and update +REDESIGN.md to describe it (per §7). Reasoning: + +- **Separation of concerns is real.** Authn (sessions, OAuth secrets, device + codes, PKCE, verification, expiry sweeps) and authz (the grant graph) have + different security surfaces and change cadences. Intermingling token/secret + tables with the grant graph — which every group/membership migration touches — + is exactly what you want to avoid, and it's consistent with the design's own + clean authz boundary (the `_tree_access` seam). +- **An established auth *shape* beats greenfield SQL auth.** The redesign's + `core.oauth_flow`/`oauth_identity`/`session` are underspecified, hand-rolled + auth. Mirroring better-auth's vetted model (account linking, multi-provider, + verification, session lifecycle) is lower-risk and leaves a credible path to + adopt the library or a managed service later. +- **The shared-id pattern neutralizes the redesign's main edge.** + `auth.users.id == core.principal.id` is the standard "identity table ↔ domain + entity share a PK" pattern: no mapping table, no meaningful duplication — two + concern-specific rows under one id, written atomically by `provisionUser` + (`packages/server/provision.ts:80,89`). You keep ~all the simplicity of "one + identity" while keeping the boundary. +- **Preserves optionality.** The deliberate absence of a cross-schema FK keeps + `auth` splittable onto its own DB/service later (`packages/database/index.ts` + notes it could be "distributed across databases again"). + +Caveats (cost of this choice): + +- It follows better-auth's **shape**, not the library — `packages/auth` depends + only on `@memory.build/database` + `postgres`, with deliberate divergences + (sha256 `token_hash`; a bespoke `device_authorization`). So the win is a vetted + schema + upgrade path, not free battle-tested code. +- `auth.verifications` is a **dead table** carried for shape parity (never + written). +- The `auth.users` ⇄ `core.principal` invariant is **app-enforced only** (no DB + FK). Whether to add one is its own decision — see + `DECISIONS_FOR_REVIEW.md` → "No cross-schema FK between `core.principal` and + `auth.users`" (current call: don't, defer to user-deletion / standalone-users). + ### B. Reserved tree paths and provisioning are built, not deferred This is the largest behavioral divergence. The redesign's V1 scope says (§"Private @@ -297,7 +334,8 @@ about or most opinionated on, and the code honors them: If REDESIGN.md is meant to track reality, these lines are now stale: - Move `core.session` / `core.oauth_identity` / `core.oauth_flow` out of the - `core` section and document the separate `auth` schema (better-auth shape). + `core` section and document the separate `auth` schema (better-auth shape) — + decided to keep the separate schema, see §2.A. - §"Private Areas" / §"me space create": the `home`/`share`/`~` convention and the creator's `owner@home`+`owner@share` (not `owner@root`) grant are shipped, not deferred — update the V1 scope. From 708c1ec6910cd9ba4596256b8762d2a5e67846f7 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 16:17:56 +0200 Subject: [PATCH 096/156] =?UTF-8?q?docs:=20adjudicate=20item=20B=20(tree?= =?UTF-8?q?=20provisioning)=20=E2=80=94=20keep=20current,=20for=20UX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mark the home/share provisioning + creator grants as a deliberate decision: implemented for multiplayer UX (usable shared/private layout out of the box), built via plain ltree grants the way the redesign preferred (no magic paths / deny rules). Note the creator gets owner@home+owner@share, not owner@root, to fold into REDESIGN.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- REDESIGN_DIFFERENCES.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index 4e7b0a8..f93c278 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -26,7 +26,7 @@ decided. Items the redesign lists but the code does **not** build live in §1b. | # | Topic | Redesign says | Implementation does | Severity | Decision | |---|-------|---------------|---------------------|----------|----------| | A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | **Keep current** (see §2.A) | -| B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | — | +| B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | **Keep current** (UX; see §2.B) | | D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | **Keep current** (see §3.D) | | E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | — | @@ -105,7 +105,7 @@ Caveats (cost of this choice): `DECISIONS_FOR_REVIEW.md` → "No cross-schema FK between `core.principal` and `auth.users`" (current call: don't, defer to user-deletion / standalone-users). -### B. Reserved tree paths and provisioning are built, not deferred +### B. Reserved tree paths and provisioning are built, not deferred — decision: **keep the current implementation** This is the largest behavioral divergence. The redesign's V1 scope says (§"Private Areas", §"me space create"): @@ -134,6 +134,20 @@ respected. But the **convention layer the redesign deferred is shipped**, and th creator's grant is `home`+`share` rather than `root`. Any reader of REDESIGN.md would expect a fresh space to be empty and root-owned; it is neither. +**Decision: keep current — implemented deliberately, for UX.** Multiplayer spaces +need a usable shared/private layout out of the box; making every new space's admin +design an access model from scratch before writing a single memory is poor +onboarding. The redesign itself calls the motivation valid and lists the +shared/private provisioning as a `me space create` **stretch goal** — we chose to +ship it. Importantly it's built the way the redesign *preferred*: ordinary +positive ltree grants over conventional `home`/`share` roots, **not** magic +private-path patterns or implicit deny rules — so the monotonic, no-deny non-goals +still hold (the access evaluator stays a plain ltree-containment check). The one +substantive thing to fold into REDESIGN.md is that the creator gets +`owner@home` + `owner@share` rather than `owner@root` (so a creator doesn't see +other members' homes; as an admin it can self-grant `owner@root` if it wants the +whole tree). + --- ## 3. Naming / shape divergences (same concept, different surface) @@ -338,7 +352,8 @@ If REDESIGN.md is meant to track reality, these lines are now stale: decided to keep the separate schema, see §2.A. - §"Private Areas" / §"me space create": the `home`/`share`/`~` convention and the creator's `owner@home`+`owner@share` (not `owner@root`) grant are shipped, not - deferred — update the V1 scope. + deferred — decided to keep (implemented for UX, see §2.B); promote the + shared/private provisioning from stretch goal to V1 scope. - §"Space": `core.space` does not yet carry embedding or placement columns (there's a TODO); embedding is hardcoded uniform. Either implement or downgrade the prose to "future." From 625f079d4ab7d15498718e987b6ea9b218b98338 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 16:22:04 +0200 Subject: [PATCH 097/156] =?UTF-8?q?docs:=20adjudicate=20item=20E=20(API=20?= =?UTF-8?q?endpoints)=20=E2=80=94=20keep=20current?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mark the two-endpoint split (+ REST /auth/*) as a deliberate decision: it encodes credential (session-only /user/rpc rejects api keys) and space-scope invariants at the endpoint level, making "agents can't manage agents/keys/spaces" structural rather than a per-method flag; REST auth is a necessity, not a divergence. §1a is now fully adjudicated (A/B/D/E all keep-current). Co-Authored-By: Claude Opus 4.8 (1M context) --- REDESIGN_DIFFERENCES.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index f93c278..fa0f297 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -28,7 +28,7 @@ decided. Items the redesign lists but the code does **not** build live in §1b. | A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | **Keep current** (see §2.A) | | B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | **Keep current** (UX; see §2.B) | | D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | **Keep current** (see §3.D) | -| E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | — | +| E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | **Keep current** (see §3.E) | ### 1b. Not implemented (gaps vs. the redesign) @@ -202,7 +202,7 @@ the weaker spec to be updated. Reasoning per axis: Per CLAUDE.md the auth gate is "non-empty `build_tree_access`." -### E. Two RPC endpoints, plus REST auth +### E. Two RPC endpoints, plus REST auth — decision: **keep the current implementation** The redesign describes "the hosted API server exposes JSON-RPC over HTTPS" as a single surface. The implementation splits it (`packages/server/router.ts:252`): @@ -214,8 +214,27 @@ single surface. The implementation splits it (`packages/server/router.ts:252`): - `/api/v1/auth/*` — REST device-flow endpoints (`device/code`, `device/token`, `device/verify`, `device/approve`, `callback/:provider`). -The split (agents can't manage agents/spaces) is a real design decision absent -from the doc. +**Decision: keep current.** The split isn't arbitrary — it encodes two orthogonal +policies as endpoint-level invariants: + +- **Credential.** `/user/rpc` is session-only — api keys are *rejected*, not just + unprivileged (`authenticate-user.ts`). So "agents can't manage agents / keys / + spaces" is **impossible by construction**: an agent can't even reach + `agent.*` / `apiKey.*` / `space.*`. On a single endpoint this would be a + per-method "session-only" flag — a privilege-escalation bug waiting for the + first forgotten flag. +- **Scope.** `/memory/rpc` is space-scoped (required `X-Me-Space`); the + `/user/rpc` methods are inherently global/pre-space (`space.list` / `space.create` + can't sit behind a space header). + +The REST `/auth/*` routes aren't really a divergence: OAuth device flow + provider +callbacks are redirect/poll/browser-shaped and can't be JSON-RPC methods — the +redesign's "JSON-RPC over HTTPS" simply omitted that auth must be REST. The cost +is two client classes (`createMemoryClient` / `createUserClient`) sharing the same +`protocol` + transport packages — minor. The single-endpoint alternative is +simpler to *document* but weaker: it demotes a structural security boundary to a +per-method flag. So this reads as the doc under-specifying, not a considered +alternative; "JSON-RPC over HTTPS" still describes the data/control plane. ### CLI verb renames (vs. the redesign's command list) @@ -365,6 +384,6 @@ If REDESIGN.md is meant to track reality, these lines are now stale: - Command list: `space alter`→`rename`, `apikey revoke`→`delete`, `agent group list`→`agent groups`; add `me agent add`; `me memory copy` and `me user group list` are unbuilt; the single-API framing should mention the two RPC endpoints - + `/api/v1/auth/*`. + + `/api/v1/auth/*` (decided to keep the split — see §3.E). - Add the **last-admin safeguard** to a "not yet implemented" list so it isn't assumed to exist. From 390508db7836c15d7fde8639dbc2e788aee364a2 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 18:02:25 +0200 Subject: [PATCH 098/156] feat(core): enforce the last-admin safeguard (effective-admin) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live space must always keep at least one effective admin — a user who is a direct admin or a member of an admin-flagged group. The enforce_last_admin trigger fires on principal_space (admin removed/demoted) and group_member (member removed from an admin group) and rejects any change leaving zero effective admins, raising SQLSTATE ME001 -> LAST_ADMIN. Checking the effective set (not just the principal_space.admin flag) closes the unrecoverable brick where a space's sole admin is an empty admin group. Covers principal.remove, the add_principal_to_space demote, removing the last member of the sole admin group, and delete_principal cascades; whole-space teardown is exempt (delete_space drops the space row first) and a select-for-update serializes concurrent removals. Adds the LAST_ADMIN protocol error code + core-error mapping, DB + server integration tests, and updates REDESIGN_DIFFERENCES (item F -> implemented) and CLAUDE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- REDESIGN_DIFFERENCES.md | 28 ++++- .../idempotent/001_principal_space.sql | 98 ++++++++++++++++ packages/engine/core/core.integration.test.ts | 107 ++++++++++++++++++ packages/protocol/errors.ts | 2 + packages/server/rpc/core-error.ts | 11 +- .../rpc/memory/management.integration.test.ts | 23 ++++ 7 files changed, 262 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a0619f8..6350814 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,7 @@ Read the relevant docs before starting work on a subsystem. - **Principal** = the union **user | agent | group** (`principal.kind` = `'u'` | `'a'` | `'g'`). The space roster (`principal_space`) holds principals. `principal.member_id` is a generated column equal to `id` for users/agents (NOT groups). - **Member** = the **user/agent** sense only — group members and api-key holders. So params split as `principalId` (roster / grants, any kind) vs `memberId` (group membership, api keys; u|a only). The space-roster surface is principal-centric (`principal.*` methods, `SpacePrincipal` type), reserving "member" for u|a. - **Space**: identified by an immutable 12-char `slug` (which is the `me_` schema name and the `X-Me-Space` value) and a renamable `name`. `me space rename` changes only the name. No org / engine / shard concepts. -- **Admin**: `principal_space.admin` is *structural* authority — roster mutations (`principal.add`/`remove`), groups, and invitations (`invite.*`) — distinct from data ownership (owner@path via `tree_access`). Enumerating the whole roster (`principal.list`) is admin-only; **any member** may `principal.resolve`/`lookup` (a targeted name↔id lookup, not enumeration). Admin transfers **transitively** through a group whose own `principal_space.admin` is true; agents are never admins. +- **Admin**: `principal_space.admin` is *structural* authority — roster mutations (`principal.add`/`remove`), groups, and invitations (`invite.*`) — distinct from data ownership (owner@path via `tree_access`). Enumerating the whole roster (`principal.list`) is admin-only; **any member** may `principal.resolve`/`lookup` (a targeted name↔id lookup, not enumeration). Admin transfers **transitively** through a group whose own `principal_space.admin` is true; agents are never admins. A space must always keep ≥1 *effective* admin (a **user** who is a direct admin or a member of an admin group — an empty admin group doesn't count) — the `enforce_last_admin` trigger on `principal_space` + `group_member` rejects any remove/demote/group-member-removal that would drop the last one (SQLSTATE `ME001` → `LAST_ADMIN`), but exempts whole-space deletion. - **Transitive membership** (Model 2): a group member gains the group's space membership, its space-admin (if the group is admin), and its tree-access grants. ## Project Structure diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index fa0f297..91a855f 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -37,10 +37,12 @@ Listed in the redesign but not built. Detail in §4. | # | Topic | Redesign wants | Status | |---|-------|----------------|--------| | C | Embedding config | Per-space model/dimension, recorded in `core.space` | Not implemented (hardcoded uniform; templated DDL only) | -| F | Last-admin safeguard | Must prevent removing/demoting the last admin | Not implemented | | G | `me memory copy`/`cp` | Listed in §"Memory Commands" | Not implemented | | H | `me user group list` | The one supported `me user` command for v1 | Not implemented | +(Item **F**, the last-admin safeguard, was previously listed here — now +**implemented**; see §6.) + The agent access-masking model (the part the doc was least confident about) **is** implemented as designed — see §6. @@ -264,9 +266,6 @@ All same-intent, different spelling: dimensions`, `core/migrate/incremental/001_space.sql:9`) nor a shard/placement column. Consistent with the "no sharding in v1" non-goal, but the per-space hooks the redesign called for aren't there yet. -- **F. Last-admin safeguard.** The redesign requires that no cascade or removal - may strip the last `principal_space.admin = true` from a space (§"Last-Admin - Safeguard", §`core.principal_space`). No such check exists in SQL or app code. - **G. `me memory copy` / `cp`.** Listed in §"Memory Commands"; `move`/`mv` exists, `copy`/`cp` does not (`commands/memory.ts`). The MCP server likewise has `me_memory_mv` but no copy tool. @@ -337,6 +336,22 @@ about or most opinionated on, and the code honors them: anywhere; hard deletes via FK `on delete cascade` plus explicit cascade functions (`remove_principal_from_space`, etc.). Matches §"Deletion and Cascading". +- **Last-admin safeguard** (was item F — now implemented, and stronger than the + redesign's wording). A space can't be left without an **effective** admin — a + *user* who is a direct admin **or** a member of an admin-flagged group. The + `enforce_last_admin` trigger fn (`core/migrate/idempotent/001_principal_space.sql`) + fires on `core.principal_space` (admin removed/demoted) **and** `core.group_member` + (member removed from an admin group), and rejects any change leaving zero + effective admins, raising SQLSTATE `ME001` → `LAST_ADMIN` (`rpc/core-error.ts`). + It covers every path uniformly — `principal.remove`, the `add_principal_to_space` + demote, removing the last member of the sole admin group, and FK cascades from + `delete_principal` (deleting an admin user/group) — and exempts whole-space + teardown (`delete_space` drops the `space` row first, so the trigger sees it gone + and skips; the `select … for update` also serializes concurrent removals). + Checking the *effective* set (not just the `principal_space.admin` flag) closes + the brick where a space's sole admin is an **empty admin group** — an + unrecoverable, ungoverned state the flag-only check would have allowed. Matches + (exceeds) §"Last-Admin Safeguard". - **Principal model.** `kind ∈ {u,a,g}`, generated `member_id` (u|a), generated `user_id`/`agent_id`/`group_id`, agent `owner_id → principal(user_id)`, name scoping (users global, agents per-owner, groups per-space). Matches §`core.principal`. @@ -385,5 +400,6 @@ If REDESIGN.md is meant to track reality, these lines are now stale: list`→`agent groups`; add `me agent add`; `me memory copy` and `me user group list` are unbuilt; the single-API framing should mention the two RPC endpoints + `/api/v1/auth/*` (decided to keep the split — see §3.E). -- Add the **last-admin safeguard** to a "not yet implemented" list so it isn't - assumed to exist. +- The **last-admin safeguard** is now implemented (SQLSTATE `ME001` / + `LAST_ADMIN`, the `enforce_last_admin` trigger) — keep §"Last-Admin Safeguard" + and note the trigger-based enforcement + the admin-group-with-no-members edge. diff --git a/packages/database/core/migrate/idempotent/001_principal_space.sql b/packages/database/core/migrate/idempotent/001_principal_space.sql index a112510..ba7132f 100644 --- a/packages/database/core/migrate/idempotent/001_principal_space.sql +++ b/packages/database/core/migrate/idempotent/001_principal_space.sql @@ -61,3 +61,101 @@ as $func$ ) $func$ language sql stable security invoker ; + +------------------------------------------------------------------------------- +-- enforce_last_admin (trigger fn on principal_space + group_member) +-- Invariant: a live space must always have at least one *effective* admin — a +-- user who is a direct admin (principal_space.admin) OR a member of an +-- admin-flagged group. Agents are never admins, and an admin-flagged group with +-- no user members does NOT count. Checking the effective set (not just the +-- principal_space.admin flag) closes the brick where a space's sole admin is an +-- empty admin group, leaving it unrecoverable. +-- +-- Guards every path that could drop the effective set, uniformly: +-- * principal_space remove/demote — incl. a group losing its admin flag, and +-- delete_principal cascades (deleting an admin user or group); +-- * group_member removal from an admin group — incl. remove_principal_from_space +-- and delete_principal cascades (a user leaving the sole admin group). +-- +-- Whole-space teardown is exempt: delete_space drops the core.space row and lets +-- the FK cascade scrub the roster, so by the time this fires the space row is +-- gone. The `for update` both detects that (no row -> skip) and serializes +-- concurrent admin removals on the same space (so two txns can't each drop a +-- different last-ish admin and race to zero). +------------------------------------------------------------------------------- +create or replace function {{schema}}.enforce_last_admin() +returns trigger +as $func$ +begin + -- group_member path: only an ADMIN group's membership can affect the effective + -- admin set; non-admin group churn (the common case) returns immediately. + if tg_table_name = 'group_member' then + if not exists + ( + select 1 + from {{schema}}.principal_space gps + where gps.principal_id = old.group_id + and gps.space_id = old.space_id + and gps.admin + ) then + return null; + end if; + end if; + + perform 1 from {{schema}}.space s where s.id = old.space_id for update; + if not found then + return null; -- space is being deleted: teardown, not a demote/removal + end if; + + if not + ( + -- a direct admin user + exists + ( + select 1 + from {{schema}}.principal_space ps + join {{schema}}.principal p on p.id = ps.principal_id + where ps.space_id = old.space_id + and ps.admin + and p.kind = 'u' + ) + -- or a user member of an admin-flagged group + or exists + ( + select 1 + from {{schema}}.group_member gm + join {{schema}}.principal_space gps + on gps.principal_id = gm.group_id and gps.space_id = gm.space_id and gps.admin + join {{schema}}.principal mp on mp.id = gm.member_id and mp.kind = 'u' + where gm.space_id = old.space_id + ) + ) then + raise exception + 'cannot leave space % without an effective admin', old.space_id + using errcode = 'ME001' + , hint = 'a space needs a user admin — direct, or via an admin group with at least one member'; + end if; + + return null; +end; +$func$ language plpgsql +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +-- principal_space: fire only when an admin row is removed or demoted (NEW can't +-- be referenced in a DELETE trigger's WHEN, hence two). group_member: fire on any +-- removal; the fn early-outs unless the group is an admin group. +create or replace trigger principal_space_keep_admin_del +after delete on {{schema}}.principal_space +for each row when (old.admin) +execute function {{schema}}.enforce_last_admin(); + +create or replace trigger principal_space_keep_admin_upd +after update on {{schema}}.principal_space +for each row when (old.admin and not new.admin) +execute function {{schema}}.enforce_last_admin(); + +create or replace trigger group_member_keep_admin_del +after delete on {{schema}}.group_member +for each row +execute function {{schema}}.enforce_last_admin(); diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts index ae4ed67..b786634 100644 --- a/packages/engine/core/core.integration.test.ts +++ b/packages/engine/core/core.integration.test.ts @@ -248,3 +248,110 @@ test("api keys: create, get, list, delete (no secret leaked)", async () => { expect(await core.getApiKey(key.id)).toBeNull(); expect(await core.listApiKeys(userId)).toHaveLength(0); }); + +// --------------------------------------------------------------------------- +// last-admin safeguard (enforce_last_admin trigger on principal_space) +// --------------------------------------------------------------------------- + +/** Assert a promise rejects with the last-admin guard's SQLSTATE (ME001). */ +async function expectLastAdmin(p: Promise) { + try { + await p; + throw new Error("expected a last-admin (ME001) rejection, but it resolved"); + } catch (e) { + expect((e as { code?: string }).code).toBe("ME001"); + } +} + +test("removing the last admin is rejected (ME001)", async () => { + // beforeEach made userId the space's sole admin. + await expectLastAdmin(core.removePrincipalFromSpace(spaceId, userId)); + // rolled back — the admin is still a member + const all = await core.listSpacePrincipals(spaceId); + expect(all.find((p) => p.id === userId)?.admin).toBe(true); +}); + +test("demoting the last admin is rejected (ME001)", async () => { + await expectLastAdmin(core.addPrincipalToSpace(spaceId, userId, false)); + const all = await core.listSpacePrincipals(spaceId); + expect(all.find((p) => p.id === userId)?.admin).toBe(true); +}); + +test("removing a non-last admin succeeds (another admin remains)", async () => { + const user2 = await v7(); + await core.createUser(user2, `admin2_${rand(8)}@example.com`); + await core.addPrincipalToSpace(spaceId, user2, true); // 2nd admin + + expect(await core.removePrincipalFromSpace(spaceId, userId)).toBe(true); + const admins = (await core.listSpacePrincipals(spaceId)).filter( + (p) => p.admin, + ); + expect(admins.map((p) => p.id)).toEqual([user2]); +}); + +test("deleting a group that is the space's only admin is rejected (ME001)", async () => { + // a fresh space whose sole effective admin is a user via an admin group + const sid = await core.createSpace(rand(12), "Group-admin Space"); + const groupId = await core.createGroup(sid, `admins_${rand(6)}`); + await core.addPrincipalToSpace(sid, groupId, true); + const member = await v7(); + await core.createUser(member, `gm_${rand(8)}@example.com`); + await core.addGroupMember(sid, groupId, member); // effective admin via the group + + await expectLastAdmin(core.deletePrincipal(groupId)); + // rolled back — the group is still the space's admin + const admins = (await core.listSpacePrincipals(sid)).filter((p) => p.admin); + expect(admins.map((p) => p.id)).toContain(groupId); +}); + +test("removing the last member of the sole admin group is rejected (ME001)", async () => { + const sid = await core.createSpace(rand(12), "Group-admin Space"); + const groupId = await core.createGroup(sid, `admins_${rand(6)}`); + await core.addPrincipalToSpace(sid, groupId, true); // group holds space-admin + const member = await v7(); + await core.createUser(member, `gm_${rand(8)}@example.com`); + await core.addGroupMember(sid, groupId, member); // sole effective admin + + // emptying the admin group leaves no effective admin + await expectLastAdmin(core.removeGroupMember(sid, groupId, member)); + expect( + (await core.listGroupMembers(sid, groupId)).map((m) => m.memberId), + ).toEqual([member]); + + // with a direct admin also present, removing the group member is fine + const direct = await v7(); + await core.createUser(direct, `direct_${rand(8)}@example.com`); + await core.addPrincipalToSpace(sid, direct, true); + expect(await core.removeGroupMember(sid, groupId, member)).toBe(true); +}); + +test("an empty admin group is not an effective admin (the brick is closed)", async () => { + const sid = await core.createSpace(rand(12), "Brick Space"); + const direct = await v7(); + await core.createUser(direct, `creator_${rand(8)}@example.com`); + await core.addPrincipalToSpace(sid, direct, true); // the only real admin + const emptyGroup = await core.createGroup(sid, `empties_${rand(6)}`); + await core.addPrincipalToSpace(sid, emptyGroup, true); // admin flag, no members + + // the empty admin group confers admin on nobody, so removing the direct admin + // would leave the space ungoverned — rejected. + await expectLastAdmin(core.removePrincipalFromSpace(sid, direct)); + const admins = (await core.listSpacePrincipals(sid)) + .filter((p) => p.admin) + .map((p) => p.id) + .sort(); + expect(admins).toEqual([direct, emptyGroup].sort()); +}); + +test("deleting the whole space is exempt from the guard (teardown)", async () => { + // a fresh space with a single admin — deleting the space drops the roster via + // FK cascade, which must NOT trip the last-admin guard. + const slug = rand(12); + const sid = await core.createSpace(slug, "Doomed Space"); + const admin = await v7(); + await core.createUser(admin, `doomed_${rand(8)}@example.com`); + await core.addPrincipalToSpace(sid, admin, true); + + expect(await core.deleteSpace(slug)).toBe(true); + expect(await core.getSpace(slug)).toBeNull(); +}); diff --git a/packages/protocol/errors.ts b/packages/protocol/errors.ts index 7579b37..3a16872 100644 --- a/packages/protocol/errors.ts +++ b/packages/protocol/errors.ts @@ -39,6 +39,8 @@ export const APP_ERROR_CODES = { FORBIDDEN: "FORBIDDEN", NOT_FOUND: "NOT_FOUND", CONFLICT: "CONFLICT", + /** Operation would leave a space without any admin (the last-admin safeguard). */ + LAST_ADMIN: "LAST_ADMIN", RATE_LIMITED: "RATE_LIMITED", VALIDATION_ERROR: "VALIDATION_ERROR", EMBEDDING_NOT_CONFIGURED: "EMBEDDING_NOT_CONFIGURED", diff --git a/packages/server/rpc/core-error.ts b/packages/server/rpc/core-error.ts index 3cf3150..89a6fca 100644 --- a/packages/server/rpc/core-error.ts +++ b/packages/server/rpc/core-error.ts @@ -5,14 +5,21 @@ import { AppError } from "./errors"; /** - * Unique → CONFLICT (e.g. duplicate name); foreign-key / check / bad-input → - * VALIDATION_ERROR; everything else propagates. + * Unique → CONFLICT (e.g. duplicate name); the last-admin guard (ME001) → + * LAST_ADMIN; foreign-key / check / bad-input → VALIDATION_ERROR; everything + * else propagates. */ function mapCoreError(e: unknown): never { const code = (e as { code?: string }).code; if (code === "23505") { throw new AppError("CONFLICT", "A record with that name already exists"); } + if (code === "ME001") { + throw new AppError( + "LAST_ADMIN", + "This would leave the space without an admin — promote another principal to admin first.", + ); + } if (code === "23503" || code === "23514" || code === "22P02") { throw new AppError( "VALIDATION_ERROR", diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts index 71f84be..450426b 100644 --- a/packages/server/rpc/memory/management.integration.test.ts +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -166,6 +166,29 @@ test("principal: list / add / remove", async () => { ).toBe(true); }); +test("last-admin safeguard: removing/demoting the sole admin → LAST_ADMIN", async () => { + // the provisioned owner is the space's only admin + await expectAppError( + call("principal.remove", { principalId: ownerId }), + "LAST_ADMIN", + ); + await expectAppError( + call("principal.add", { principalId: ownerId, admin: false }), + "LAST_ADMIN", + ); + + // promote a second admin, then the owner can be removed + const other = await makeUser(); + await call("principal.add", { principalId: other, admin: true }); + expect( + ( + await call<{ removed: boolean }>("principal.remove", { + principalId: ownerId, + }) + ).removed, + ).toBe(true); +}); + test("principal.resolve / lookup are available to non-admin members (list is admin-only)", async () => { const email = `target_${rand(8)}@example.com`; const targetId = await makeUserWithEmail(email); From e57b8e3aa72c342c5216e598d5ce4a5d6f80dac5 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 18:10:54 +0200 Subject: [PATCH 099/156] feat(cli): add `me group mine` to list your own groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit whoami + group.listForMember (already self-service for your own id). Records the decision in REDESIGN_DIFFERENCES: item H moves to §1a as "Keep current" — `me group mine` serves the common self case; the `me user` CLI surface for listing another user's groups stays deferred (admins can use the RPC). Co-Authored-By: Claude Opus 4.8 (1M context) --- REDESIGN_DIFFERENCES.md | 7 +++---- packages/cli/commands/group.ts | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index 91a855f..5ff4697 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -29,6 +29,7 @@ decided. Items the redesign lists but the code does **not** build live in §1b. | B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | **Keep current** (UX; see §2.B) | | D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | **Keep current** (see §3.D) | | E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | **Keep current** (see §3.E) | +| H | User group listing | `me user group list ` (the one `me user` command) | `me group mine` lists your own groups (`whoami` + `group.listForMember`); no `me user` CLI for another user (the RPC already allows it for a space admin) | Partial | **Keep current** — `me group mine` serves the common (self) case; the `me user` surface stays deferred (admins can use the RPC for others) | ### 1b. Not implemented (gaps vs. the redesign) @@ -38,10 +39,10 @@ Listed in the redesign but not built. Detail in §4. |---|-------|----------------|--------| | C | Embedding config | Per-space model/dimension, recorded in `core.space` | Not implemented (hardcoded uniform; templated DDL only) | | G | `me memory copy`/`cp` | Listed in §"Memory Commands" | Not implemented | -| H | `me user group list` | The one supported `me user` command for v1 | Not implemented | (Item **F**, the last-admin safeguard, was previously listed here — now -**implemented**; see §6.) +**implemented**; see §6. Item **H**, user group listing, moved to §1a — it's +partly built via `me group mine`.) The agent access-masking model (the part the doc was least confident about) **is** implemented as designed — see §6. @@ -269,8 +270,6 @@ All same-intent, different spelling: - **G. `me memory copy` / `cp`.** Listed in §"Memory Commands"; `move`/`mv` exists, `copy`/`cp` does not (`commands/memory.ts`). The MCP server likewise has `me_memory_mv` but no copy tool. -- **H. `me user group list `.** The redesign names this the single - supported `me user` command for v1; there is **no `me user` command** at all. - **Verified-email enforcement on invite acceptance.** The redesign (§`core.space_invitation`) wants acceptance to require an OAuth-verified email matching the invitation ("possession of an invite link alone should not be sufficient"). Invitations diff --git a/packages/cli/commands/group.ts b/packages/cli/commands/group.ts index 1010b70..9ff7702 100644 --- a/packages/cli/commands/group.ts +++ b/packages/cli/commands/group.ts @@ -5,6 +5,7 @@ * everyone in the group. Group membership also confers space membership. * * - me group list: list groups + * - me group mine: list the groups you are in * - me group create : create a group * - me group rename : rename a group * - me group delete : delete a group @@ -27,6 +28,7 @@ import { } from "../output.ts"; import { buildMemoryClient, + buildUserClient, handleError, requireSession, requireSpace, @@ -263,11 +265,45 @@ function createGroupMembersCommand(): Command { }); } +function createGroupMineCommand(): Command { + return new Command("mine") + .description("list the groups you are in (in the active space)") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); + try { + const me = await user.whoami(); + const { groups } = await memory.group.listForMember({ + memberId: me.id, + }); + output({ groups }, fmt, () => { + if (groups.length === 0) { + console.log(" You're not in any groups in this space."); + return; + } + table( + ["group", "admin", "id"], + groups.map((g) => [g.name, g.admin ? "yes" : "", g.groupId]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + export function createGroupCommand(): Command { const group = new Command("group").description( "manage groups in the active space", ); group.addCommand(createGroupListCommand()); + group.addCommand(createGroupMineCommand()); group.addCommand(createGroupCreateCommand()); group.addCommand(createGroupRenameCommand()); group.addCommand(createGroupDeleteCommand()); From 404accca6539b3151a3e8fd3f337b819d32343b9 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Sat, 6 Jun 2026 18:16:43 +0200 Subject: [PATCH 100/156] feat(cli): accept `revoke` as an alias for `me apikey delete` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Back-compat / familiarity (the redesign's verb). `delete` stays canonical (revoke ≡ delete, no soft-delete state); `rm` and `revoke` are aliases. Co-Authored-By: Claude Opus 4.8 (1M context) --- REDESIGN_DIFFERENCES.md | 7 ++++--- packages/cli/commands/apikey.ts | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md index 5ff4697..ca29e74 100644 --- a/REDESIGN_DIFFERENCES.md +++ b/REDESIGN_DIFFERENCES.md @@ -245,9 +245,10 @@ All same-intent, different spelling: - `me space alter` → **`me space rename`** (`commands/space.ts:212`). - `me agent group list` → **`me agent groups`** (`commands/agent.ts:161`). -- `me apikey revoke` → **`me apikey delete`/`rm`** (+ a `me apikey get`) - (`commands/apikey.ts`). The rename aligns with the doc's own "no soft delete / - hard delete" stance, but the doc text still says `revoke`. +- `me apikey revoke` → **`me apikey delete`** (aliases `rm`, `revoke`) (+ a + `me apikey get`) (`commands/apikey.ts`). Canonical verb is `delete` (aligning + with the "no soft delete / hard delete" stance — revoke ≡ delete); `revoke` is + kept as an alias for familiarity. - `me group member add/remove/list` → **`me group add` / `remove`(`rm-member`) / `members`** (`commands/group.ts`). diff --git a/packages/cli/commands/apikey.ts b/packages/cli/commands/apikey.ts index 804bd61..f249508 100644 --- a/packages/cli/commands/apikey.ts +++ b/packages/cli/commands/apikey.ts @@ -125,7 +125,7 @@ function createApiKeyGetCommand(): Command { function createApiKeyDeleteCommand(): Command { return new Command("delete") - .alias("rm") + .aliases(["rm", "revoke"]) .description("delete (revoke) an API key") .argument("", "API key id") .option("-y, --yes", "skip confirmation prompt") From 9516225d5eec6b8c0c3b066bd40fded59c503451 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 8 Jun 2026 11:02:51 +0200 Subject: [PATCH 101/156] test: run the suite against ghost by default, parallelized (--parallel=2) `./bun run test` (and `check`) now default TEST_DATABASE_URL to the ghost instance and run files via bun's --parallel=2 with a 30s timeout, so the whole suite (unit + integration) validates against the canonical Postgres in ~4 min; set TEST_DATABASE_URL (e.g. a local Postgres) to override, and raw `bun test ` is unchanged (local default). test:db broadens to every *.integration.test.ts under packages/ (was packages/database only) and likewise uses --parallel=2. Update CLAUDE.md. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 45 ++++++++++++++++++++++++++++++--------------- package.json | 4 ++-- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6350814..63055f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,17 +82,22 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): # Linting and formatting (auto-fix) ./bun run lint --write -# Run unit tests -./bun test - -# Run a single test file +# Run a test file directly (uses TEST_DATABASE_URL; default local 127.0.0.1) — +# fast, for iterating on one file: ./bun test packages/cli/mcp/install.test.ts -# Shorthand for all checks (typecheck + lint + test) +# Full suite (unit + integration). `test` and `check` default TEST_DATABASE_URL +# to the ghost instance and run files in parallel (--parallel=2) with a 30s +# timeout; set TEST_DATABASE_URL (e.g. a local Postgres) to override. +./bun run test + +# Shorthand for all checks (typecheck + lint + test → ghost) ./bun run check ``` -**Important**: After making code changes, always run `./bun run check`. +**Important**: After making code changes, always run `./bun run check` (it runs +the full suite against ghost, ~4 min; for a faster inner loop set +`TEST_DATABASE_URL` to a local Postgres). > `packages/web` and `packages/docs-site` are excluded from the root typecheck > (they have their own); `./bun run check` does not cover them. @@ -101,7 +106,12 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): `*.integration.test.ts` files run against a real PostgreSQL 18 with the required extensions (citext, ltree, pgvector, pg_textsearch), provisioned with -ghost. Point `TEST_DATABASE_URL` at a ghost database and run: +ghost. `./bun run test` and `check` already target ghost by default (the full +suite, `--parallel=2`, 30s timeout — see above). `test:db` is the focused +variant: it first reclaims orphaned test schemas, then runs **every** +`*.integration.test.ts` under `packages/` (the auth/core/space migration suites +plus the engine/server/worker suites), `--parallel=2`, 30s timeout. Point +`TEST_DATABASE_URL` at a ghost database and run: ```bash TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db @@ -121,19 +131,24 @@ TEST_DATABASE_URL="$(ghost connect testing_me)" \ Isolation is **schema-level** (ghost forbids `create database`): each test provisions its own throwaway schema(s) — `core_test_` for core, -`auth_test_` for auth, `metest_` for spaces — so the suites are -fully concurrent and parallel-safe across files. All migrations are templated, -so production uses `core` / `auth` / `me_` while tests target throwaway -schemas and never touch real data. Test spaces deliberately use the `metest_` -prefix (not the production `me_`) so leftovers are distinguishable from real -spaces by name alone. +`auth_test_` for auth, `metest_` for the space *migration* tests — so +the suites are fully concurrent and parallel-safe across files. All migrations are +templated, so production uses `core` / `auth` / `me_` while these tests +target throwaway schemas and never touch real data. The space migration tests +deliberately use the `metest_` prefix (not production `me_`) so leftovers are +distinguishable by name alone. The **server** integration tests are the exception: +they exercise the real `provisionUser` / `provisionSpace`, so they create genuine +`me_` schemas and drop them in `afterAll` (only a hard-interrupted server +test leaks one — see below). `test:db` first runs `test:db:clean` (`scripts/clean-test-schemas.ts`) to reclaim orphaned `core_test_*` / `auth_test_*` / `metest_*` schemas left by hard-interrupted runs. It is age-gated (only drops schemas older than 60 min, so a concurrent `test:db` sharing the database is safe) and a no-op against a -production database — no pattern can match a real schema. Use `test:db:clean:all` -for a deliberate full reset when nothing else is using the database. +production database — no pattern can match a real schema, **including `me_`**: +a server test's leaked `me_` is therefore *not* auto-reclaimed, so drop it +by hand if a run is killed mid-test. Use `test:db:clean:all` for a deliberate full +reset when nothing else is using the database. ## Style Guides diff --git a/package.json b/package.json index 2d25bf9..da21d89 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,8 @@ "release:server": "./bun scripts/release-server.ts", "server": "./bun run packages/server/index.ts", "setup": "./bun scripts/setup.ts", - "test": "./bun test packages", - "test:db": "./bun run test:db:clean && find packages/database -name '*.integration.test.ts' -print0 | xargs -0 -P 2 -n 1 ./bun test --timeout 30000", + "test": "TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-$(ghost connect testing_me)}\" ./bun test packages --timeout 30000 --parallel=2", + "test:db": "./bun run test:db:clean && find packages -name '*.integration.test.ts' -print0 | xargs -0 ./bun test --parallel=2 --timeout 30000", "test:db:clean": "./bun scripts/clean-test-schemas.ts", "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", "typecheck": "tsc --noEmit" From f976541341151bbf525bac9abea9748666586246 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 8 Jun 2026 12:03:35 +0200 Subject: [PATCH 102/156] =?UTF-8?q?test(e2e):=20real=20CLI=E2=86=92server?= =?UTF-8?q?=E2=86=92ghost=20end-to-end=20suite=20(+=20enabling=20fixes)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an e2e tier that drives the real `me` binary as a subprocess against a real in-process server and the ghost test DB, with real OpenAI embeddings: - Phase 1: extract a side-effect-free `startServer()` (packages/server/start.ts, exported from lib.ts); index.ts is now a thin entrypoint that owns telemetry + signal handlers. Fix schema threading so migrations run against the configured auth/core schemas. Add start.integration.test.ts. - Phase 2: make the space-schema prefix configurable (SPACE_SCHEMA_PREFIX, lazy read, default me_); the harness uses metest_ so the existing reclaimer sweeps e2e spaces. - Phase 3: e2e/cli.e2e.test.ts (new @memory.build/e2e workspace) + test:e2e script. 8/8 scenarios: whoami, create/tree, BM25 + semantic search, tree conventions, update/delete, api-key auth, failure modes. Two bugs the e2e suite surfaced (fixed): - treePathSchema was stricter than the server's own normalizeTreePath, rejecting ~/share/slash (and hyphenated) paths over the wire. Relax it to the lenient input form; the server still normalizes + validates authoritatively. - CLI memory commands were session-only, ignoring ME_API_KEY (contradicting the docs). Add requireMemoryAuth and make buildMemoryClient prefer the api key, across the memory data-plane commands. Record one open decision (default agent grant on join) in DECISIONS_FOR_REVIEW.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- DECISIONS_FOR_REVIEW.md | 39 ++ bun.lock | 12 + e2e/cli.e2e.test.ts | 335 ++++++++++++++++ e2e/package.json | 12 + package.json | 4 +- packages/cli/commands/import.ts | 4 +- packages/cli/commands/memory-import.ts | 4 +- packages/cli/commands/memory.ts | 20 +- packages/cli/commands/pack.ts | 6 +- packages/cli/util.ts | 37 +- packages/database/space/slug.test.ts | 28 +- packages/database/space/slug.ts | 25 +- packages/protocol/fields.ts | 21 +- packages/server/index.ts | 451 +-------------------- packages/server/lib.ts | 6 + packages/server/start.integration.test.ts | 88 +++++ packages/server/start.ts | 453 ++++++++++++++++++++++ 17 files changed, 1078 insertions(+), 467 deletions(-) create mode 100644 e2e/cli.e2e.test.ts create mode 100644 e2e/package.json create mode 100644 packages/server/start.integration.test.ts create mode 100644 packages/server/start.ts diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index d547b4f..8afe161 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -124,6 +124,45 @@ making them mintable for users widens that surface. --- +## Should an agent get `share` access on join by default, or no grants (as now)? + +**Date:** 2026-06-08 · **Area:** membership (`me agent add` / `principal.add`) + +Surfaced by the e2e suite: `me agent add` puts the agent on the roster but +grants it **nothing**, so a freshly-added agent (with a minted key) gets +`No access to this space` on its first `me search` — the auth gate is a +non-empty `build_tree_access`, and an agent joins with zero grants (see the +"Home grant at join is for users only" entry: agents get no auto home because +the `agent_tree_access` clamp would make it inert). To make the agent usable the +owner must run an explicit `me access grant share r` (or similar) after +adding it. The e2e api-key scenario does exactly that. + +**The decision:** when an agent is added to a space, should it automatically +receive a default grant — most naturally **read on `share`**, the shared root — +so it's immediately usable, or should it keep getting **no grants** (today), +requiring the owner to grant access explicitly? + +**Why it's a real decision:** weigh ergonomics (an added agent that can do +nothing until a second, easily-forgotten grant command is surprising) against +least-privilege (an agent should see only what its owner deliberately shares). +Note the clamp: an agent's effective access is bounded by its owner's, so a +default `read@share` would only take effect when the owner themselves can read +`share` (the space creator owns it; an invited member may or may not). A default +also raises "which level/path" (read vs write, `share` vs space-root) and whether +it should apply to all join paths (`principal.add`, invite redemption) or only +self-service `me agent add`. + +**How to change it (add a default):** in `add_principal_to_space` +(`packages/database/core/migrate/idempotent/006_membership.sql`) add an +agent-branch that writes a `read @ share` grant (mirroring the user home-grant +branch gated on `p.kind = 'u'`), or do it at the RPC layer in `principal.add` +(`packages/server/rpc/memory/principal.ts`). Keeping it in the SQL chokepoint +makes it uniform across every join path. + +**Status:** needs decision. + +--- + ## No cross-schema FK between `core.principal` and `auth.users` **Date:** 2026-06-06 · **Area:** auth / core schema boundary diff --git a/bun.lock b/bun.lock index 0291073..348d72d 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,16 @@ "typescript": "^6.0.2", }, }, + "e2e": { + "name": "@memory.build/e2e", + "version": "0.2.5", + "dependencies": { + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", + "@memory.build/embedding": "workspace:*", + "postgres": "^3.4.9", + }, + }, "packages/auth": { "name": "@memory.build/auth", "version": "0.0.0", @@ -389,6 +399,8 @@ "@memory.build/docs-site": ["@memory.build/docs-site@workspace:packages/docs-site"], + "@memory.build/e2e": ["@memory.build/e2e@workspace:e2e"], + "@memory.build/embedding": ["@memory.build/embedding@workspace:packages/embedding"], "@memory.build/engine": ["@memory.build/engine@workspace:packages/engine"], diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts new file mode 100644 index 0000000..f47d8bb --- /dev/null +++ b/e2e/cli.e2e.test.ts @@ -0,0 +1,335 @@ +// End-to-end CLI integration test. +// +// me (spawned binary) ──HTTP──▶ server (real Bun.serve) ──▶ postgres.js ──▶ ghost test DB +// +// Drives the real `me` CLI as a subprocess against a real in-process server +// (startServer) and the ghost test database, with real OpenAI embeddings. No +// mocks between the CLI and the database. See PLAN_E2E_TESTING.md. +// +// TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:e2e +// +// Boundaries (deliberate): authentication is token injection (provisionUser + +// createSession → ME_SESSION_TOKEN), not `me login`; embeddings hit real OpenAI +// directly (a key is required to run — the suite skips, not fails, without one). + +// Set the space-schema prefix before anything reads it. slugToSchema reads +// SPACE_SCHEMA_PREFIX lazily (per call), so this only needs to run before the +// first call at runtime (in beforeAll) — well after import hoisting. Spaces +// land under metest_ so the existing reclaimer sweeps them. +process.env.SPACE_SCHEMA_PREFIX = "metest_"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { Sql } from "postgres"; +import { + connect, + resolveTestDatabaseUrl, +} from "../packages/database/migrate/test-utils.ts"; +import { type RunningServer, startServer } from "../packages/server/lib.ts"; +import { provisionUser } from "../packages/server/provision.ts"; + +const OPENAI_KEY = process.env.OPENAI_API_KEY ?? process.env.EMBEDDING_API_KEY; + +const repoRoot = join(import.meta.dir, ".."); +const CLI = join(repoRoot, "packages/cli/index.ts"); + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; // harness's own connection (setup/teardown/assert) +let srv: RunningServer; +let authSchema: string; +let coreSchema: string; +let spaceSlug: string; +let token: string; +let tmpHome: string; + +describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( + "cli e2e", + () => { + beforeAll(async () => { + sql = connect(); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); + + // Provision the user (and its default space) BEFORE booting the server, so + // the worker discovers the space at startup — no rediscovery lag for the + // initial space. + const provisioned = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: "e2e@example.test", + name: "E2E", + provider: "github", + accountId: `e2e-${rand()}`, + emailVerified: true, + }, + ); + spaceSlug = provisioned.spaceSlug; + ({ token } = await authStore(sql, authSchema).createSession( + provisioned.userId, + )); + + const embeddingConfig: EmbeddingConfig = { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + // OPENAI_KEY is non-null here (describe.skipIf guards it). + apiKey: OPENAI_KEY as string, + options: {}, + }; + + srv = await startServer({ + port: 0, + databaseUrl: resolveTestDatabaseUrl(), + apiBaseUrl: "http://localhost", // OAuth callbacks unused (token injection) + authSchema, + coreSchema, + migrate: false, // harness already migrated + enableCleanupCron: false, + workerCount: 1, + workerIdleDelayMs: 250, // poll the embed queue fast + workerRefreshIntervalMs: 500, // discover new spaces fast + embeddingConfig, + }); + + tmpHome = await mkdtemp(join(tmpdir(), "me-e2e-")); + }); + + afterAll(async () => { + await srv?.stop(); + // Drop the space schemas this run created (enumerating core.space covers + // CLI-created spaces too), then the auth/core test schemas. + if (sql && coreSchema) { + const spaces = await sql.unsafe(`select slug from ${coreSchema}.space`); + for (const row of spaces) { + await sql.unsafe(`drop schema if exists metest_${row.slug} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); + } + if (tmpHome) await rm(tmpHome, { recursive: true, force: true }); + }); + + // ------------------------------------------------------------------------- + // CLI subprocess helpers + // ------------------------------------------------------------------------- + + function cliEnv( + extra: Record = {}, + ): Record { + const env = { ...process.env } as Record; + // Curate: drop any ambient ME_* so the dev's shell can't leak in. + for (const k of [ + "ME_API_KEY", + "ME_SERVER", + "ME_SPACE", + "ME_SESSION_TOKEN", + ]) { + delete env[k]; + } + return { + ...env, + HOME: tmpHome, + XDG_CONFIG_HOME: join(tmpHome, ".config"), + ME_NO_KEYCHAIN: "1", + ME_SERVER: srv.url, + ME_SESSION_TOKEN: token, + ME_SPACE: spaceSlug, + ...extra, + }; + } + + async function me( + args: string[], + extraEnv?: Record, + ): Promise<{ stdout: string; stderr: string; code: number }> { + const proc = Bun.spawn([process.execPath, CLI, ...args], { + env: cliEnv(extraEnv), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const code = await proc.exited; + return { stdout, stderr, code }; + } + + // Parse the --json stdout of a `me` invocation, asserting success. + async function meJson( + args: string[], + extraEnv?: Record, + ): Promise { + const r = await me([...args, "--json"], extraEnv); + expect( + r.code, + `expected exit 0 for \`me ${args.join(" ")}\`\nstdout: ${r.stdout}\nstderr: ${r.stderr}`, + ).toBe(0); + return JSON.parse(r.stdout) as T; + } + + // Poll the space schema until N memories have a non-null embedding. + async function waitForEmbeddings(count: number, timeoutMs = 30000) { + const schema = `metest_${spaceSlug}`; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const [row] = await sql.unsafe( + `select count(*)::int as n from ${schema}.memory where embedding is not null`, + ); + if ((row?.n ?? 0) >= count) return; + await Bun.sleep(250); + } + throw new Error(`timed out waiting for ${count} embeddings`); + } + + // ------------------------------------------------------------------------- + // Core scenarios + // ------------------------------------------------------------------------- + + test("1. whoami reports the provisioned identity", async () => { + const r = await me(["whoami"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("e2e@example.test"); + }); + + test("2. create + tree round-trip (default share namespace)", async () => { + const created = await meJson<{ id: string; tree?: string }>([ + "create", + "the quick brown fox jumps over the lazy dog", + ]); + expect(created.id).toBeTruthy(); + + const r = await me(["memory", "tree"]); + expect(r.code).toBe(0); + expect(r.stdout.toLowerCase()).toContain("share"); + }); + + test("3. fulltext (BM25) search finds the memory", async () => { + const res = await meJson<{ + total: number; + results: { id: string; content: string }[]; + }>(["search", "--fulltext", "fox"]); + expect(res.total).toBeGreaterThan(0); + expect( + res.results.some((m) => m.content.includes("quick brown fox")), + ).toBe(true); + }); + + test("4. semantic search ranks a paraphrase near the top", async () => { + // Seed a few more memories to make ranking meaningful. + await meJson(["create", "a dog chased a cat across the yard"]); + await meJson(["create", "the stock market fell sharply on Tuesday"]); + await meJson(["create", "photosynthesis converts sunlight into energy"]); + + // 4 created so far in `share` (1 from scenario 2 + 3 here). Wait for the + // worker to embed them. + await waitForEmbeddings(4); + + const res = await meJson<{ + results: { id: string; content: string }[]; + }>(["search", "--semantic", "wild canine leaps over a sleepy hound"]); + // Recall-based: the fox/dog memories should surface near the top, not the + // stock-market or photosynthesis ones. Assert a relevant item is in top-3. + const top3 = res.results.slice(0, 3).map((m) => m.content); + expect(top3.some((c) => c.includes("fox") || c.includes("dog"))).toBe( + true, + ); + }); + + test("5. tree paths reflect ~ (home) and share conventions", async () => { + await meJson(["create", "personal note", "--tree", "~/notes"]); + await meJson(["create", "team note", "--tree", "share/team"]); + + const r = await me(["memory", "tree"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("notes"); + expect(r.stdout).toContain("team"); + }); + + test("6. update + delete round-trip", async () => { + const created = await meJson<{ id: string }>([ + "create", + "ephemeral memory to edit", + ]); + const updated = await meJson<{ id: string; content: string }>([ + "memory", + "update", + created.id, + "--content", + "edited content", + ]); + expect(updated.content).toBe("edited content"); + + const del = await me(["memory", "delete", created.id, "--yes"]); + expect(del.code).toBe(0); + + // Getting it now fails with a non-zero exit. + const get = await me(["memory", "get", created.id]); + expect(get.code).not.toBe(0); + }); + + // ------------------------------------------------------------------------- + // Extended scenarios + // ------------------------------------------------------------------------- + + test("7. api-key auth works end-to-end (no session token)", async () => { + // Mint the key through the real CLI: create the agent, add it to the + // space, then mint a key for it. + const agent = await meJson<{ id: string }>([ + "agent", + "create", + `bot-${rand()}`, + ]); + await me(["agent", "add", agent.id]); // bring the agent into the space + // Agents join with no grant (their access is clamped to the owner's), so + // grant read on `share` — where the fox memory lives — to make it readable. + await meJson(["access", "grant", agent.id, "share", "r"]); + const key = await meJson<{ id: string; key: string }>([ + "apikey", + "create", + agent.id, + ]); + expect(key.key).toMatch(/^me\./); + + // Search with ONLY the api key — no session token. The agent's global key + // plus X-Me-Space (ME_SPACE) selects the space; this exercises the CLI's + // api-key auth path against the real server end-to-end. + const res = await meJson<{ total: number }>( + ["search", "--fulltext", "fox"], + { ME_API_KEY: key.key, ME_SESSION_TOKEN: "" }, + ); + expect(res.total).toBeGreaterThan(0); + }); + + test("10. failure modes: bad space and missing auth exit non-zero", async () => { + const badSpace = await me(["search", "--fulltext", "fox"], { + ME_SPACE: "doesnotexist1", + }); + expect(badSpace.code).not.toBe(0); + + const noAuth = await me(["whoami"], { ME_SESSION_TOKEN: "" }); + expect(noAuth.code).not.toBe(0); + }); + }, +); diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 0000000..af57ff7 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,12 @@ +{ + "name": "@memory.build/e2e", + "version": "0.2.5", + "private": true, + "type": "module", + "dependencies": { + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", + "@memory.build/embedding": "workspace:*", + "postgres": "^3.4.9" + } +} diff --git a/package.json b/package.json index da21d89..05657c5 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "private": true, "workspaces": [ "packages/*", - "scripts" + "scripts", + "e2e" ], "scripts": { "build": "./bun run --filter '@memory.build/cli' build", @@ -34,6 +35,7 @@ "test:db": "./bun run test:db:clean && find packages -name '*.integration.test.ts' -print0 | xargs -0 ./bun test --parallel=2 --timeout 30000", "test:db:clean": "./bun scripts/clean-test-schemas.ts", "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", + "test:e2e": "./bun run test:db:clean && ./bun test --timeout 60000 ./e2e", "typecheck": "tsc --noEmit" }, "devDependencies": { diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index b04ee9d..54ccc3e 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -37,7 +37,7 @@ import { getOutputFormat, output } from "../output.ts"; import { buildMemoryClient, handleError, - requireSession, + requireMemoryAuth, requireSpace, } from "../util.ts"; @@ -164,7 +164,7 @@ async function runAndRender( typeof globalOpts.server === "string" ? globalOpts.server : undefined, ); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); let config: ReturnType; diff --git a/packages/cli/commands/memory-import.ts b/packages/cli/commands/memory-import.ts index 8a9b115..8b52a4d 100644 --- a/packages/cli/commands/memory-import.ts +++ b/packages/cli/commands/memory-import.ts @@ -18,7 +18,7 @@ import { type ParsedMemory, parseContent, } from "../parsers/index.ts"; -import { buildMemoryClient, requireSession, requireSpace } from "../util.ts"; +import { buildMemoryClient, requireMemoryAuth, requireSpace } from "../util.ts"; /** * Collect files from a path. If directory, requires --recursive. @@ -101,7 +101,7 @@ export function createMemoryImportCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); // Validate format option diff --git a/packages/cli/commands/memory.ts b/packages/cli/commands/memory.ts index 2107fbe..77038a1 100644 --- a/packages/cli/commands/memory.ts +++ b/packages/cli/commands/memory.ts @@ -23,7 +23,7 @@ import { getOutputFormat, output, table } from "../output.ts"; import { buildMemoryClient, handleError, - requireSession, + requireMemoryAuth, requireSpace, } from "../util.ts"; import { editMemory } from "./memory-edit.ts"; @@ -107,7 +107,7 @@ function createMemoryCreateCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); // Resolve content: positional > --content flag > stdin @@ -162,7 +162,7 @@ function createMemoryGetCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); const client = buildMemoryClient(creds); @@ -235,7 +235,7 @@ function createMemorySearchCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); // Resolve search text. A positional query runs hybrid search @@ -371,7 +371,7 @@ function createMemoryUpdateCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); // Resolve content @@ -424,7 +424,7 @@ function createMemoryDeleteCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); const client = buildMemoryClient(creds); @@ -501,7 +501,7 @@ function createMemoryEditCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); const client = buildMemoryClient(creds); @@ -523,7 +523,7 @@ function createMemoryTreeCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); const client = buildMemoryClient(creds); @@ -558,7 +558,7 @@ function createMemoryMoveCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); const client = buildMemoryClient(creds); @@ -654,7 +654,7 @@ function createMemoryExportCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); const format = opts.format as "json" | "yaml" | "md"; diff --git a/packages/cli/commands/pack.ts b/packages/cli/commands/pack.ts index 7f4296e..ac4a553 100644 --- a/packages/cli/commands/pack.ts +++ b/packages/cli/commands/pack.ts @@ -16,7 +16,7 @@ import { parsePack, validatePackConstraints } from "../parsers/pack.ts"; import { buildMemoryClient, handleError, - requireSession, + requireMemoryAuth, requireSpace, } from "../util.ts"; @@ -103,7 +103,7 @@ function createPackInstallCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); try { @@ -366,7 +366,7 @@ function createPackListCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); + requireMemoryAuth(creds, fmt); requireSpace(creds, fmt); const client = buildMemoryClient(creds); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 14d4230..e9116ee 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -19,6 +19,10 @@ const UUIDV7_RE = /** * Ensure the user has a session token. Exits with an error if not. + * + * Use this for the user endpoint (/api/v1/user/rpc), which is session-only — an + * api key never authenticates there (agents can't manage agents). For the memory + * endpoint, which accepts either bearer, use {@link requireMemoryAuth}. */ export function requireSession( creds: ResolvedCredentials, @@ -34,6 +38,29 @@ export function requireSession( } } +/** + * Ensure the caller can authenticate to the memory endpoint + * (/api/v1/memory/rpc), which accepts either bearer: an agent api key + * (ME_API_KEY) or a human session token. Exits with an error if neither is + * present. Pair with {@link requireSpace}; then {@link buildMemoryClient} picks + * the bearer (api key first, mirroring `me mcp`). + */ +export function requireMemoryAuth( + creds: ResolvedCredentials, + fmt: OutputFormat, +): void { + if (!creds.apiKey && !creds.sessionToken) { + const msg = + "Not authenticated. Run 'me login', or set ME_API_KEY for an agent."; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); + } +} + /** * Ensure an active space (the X-Me-Space) is selected. Exits with an error if * not. Used by the space-scoped commands (memory, group, access, …). @@ -65,15 +92,17 @@ export function buildUserClient( } /** - * Build a memory client (session + active space, /api/v1/memory/rpc). Call - * requireSession and requireSpace first so both are present. + * Build a memory client (bearer + active space, /api/v1/memory/rpc). Call + * requireMemoryAuth and requireSpace first so a bearer and a space are present. + * The bearer is the agent api key when set (ME_API_KEY), else the session token + * — the memory endpoint accepts either, and this mirrors `me mcp`'s precedence. */ export function buildMemoryClient( - creds: ResolvedCredentials & { sessionToken: string; activeSpace: string }, + creds: ResolvedCredentials & { activeSpace: string }, ): MemoryClient { return createMemoryClient({ url: creds.server, - token: creds.sessionToken, + token: creds.apiKey ?? creds.sessionToken, space: creds.activeSpace, }); } diff --git a/packages/database/space/slug.test.ts b/packages/database/space/slug.test.ts index bc31008..615f86e 100644 --- a/packages/database/space/slug.test.ts +++ b/packages/database/space/slug.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { randomSlug } from "./migrate/test-utils"; import { isValidSlug, @@ -59,6 +59,32 @@ describe("slugToSchema / schemaToSlug", () => { }); }); +describe("SPACE_SCHEMA_PREFIX override", () => { + const ORIGINAL = process.env.SPACE_SCHEMA_PREFIX; + afterEach(() => { + if (ORIGINAL === undefined) delete process.env.SPACE_SCHEMA_PREFIX; + else process.env.SPACE_SCHEMA_PREFIX = ORIGINAL; + }); + + test("slugToSchema / schemaToSlug honor a non-default prefix", () => { + process.env.SPACE_SCHEMA_PREFIX = "metest_"; + expect(slugToSchema("abcdef012345")).toBe("metest_abcdef012345"); + expect(schemaToSlug("metest_abcdef012345")).toBe("abcdef012345"); + expect(schemaToSlug(slugToSchema("0a1b2c3d4e5f"))).toBe("0a1b2c3d4e5f"); + }); + + test("isValidSpaceSchema matches the configured prefix, not me_", () => { + process.env.SPACE_SCHEMA_PREFIX = "metest_"; + expect(isValidSpaceSchema("metest_abcdef012345")).toBe(true); + expect(isValidSpaceSchema("me_abcdef012345")).toBe(false); + }); + + test("rejects an invalid prefix", () => { + process.env.SPACE_SCHEMA_PREFIX = "9bad"; + expect(() => slugToSchema("abcdef012345")).toThrow(); + }); +}); + describe("randomSlug", () => { test("always produces a valid, schema-safe slug", () => { for (let i = 0; i < 1000; i++) { diff --git a/packages/database/space/slug.ts b/packages/database/space/slug.ts index 557f7d1..7dc4200 100644 --- a/packages/database/space/slug.ts +++ b/packages/database/space/slug.ts @@ -1,6 +1,23 @@ -const SPACE_SCHEMA_RE = /^me_[a-z0-9]{12}$/; const SLUG_RE = /^[a-z0-9]{12}$/; const SLUG_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +const DEFAULT_SPACE_PREFIX = "me_"; + +// The space-schema prefix is configurable via SPACE_SCHEMA_PREFIX (default +// "me_"), mirroring AUTH_SCHEMA/CORE_SCHEMA. It is read lazily (per call) rather +// than at module load so a test can set the env before the first call despite +// import hoisting (the e2e harness sets "metest_" so its spaces are swept by the +// existing schema reclaimer). Production leaves it unset → "me_". +function spacePrefix(): string { + const p = process.env.SPACE_SCHEMA_PREFIX ?? DEFAULT_SPACE_PREFIX; + // Must be a SQL-identifier-safe prefix: lowercase letters/digits/underscore, + // starting with a letter and ending with "_". + if (!/^[a-z][a-z0-9_]*_$/.test(p)) { + throw new Error( + `Invalid SPACE_SCHEMA_PREFIX: "${p}" — must be lowercase [a-z0-9_], start with a letter, and end with "_"`, + ); + } + return p; +} /** Generate a random 12-char lowercase-alphanumeric space slug. */ export function generateSlug(): string { @@ -11,7 +28,7 @@ export function generateSlug(): string { } export function isValidSpaceSchema(name: string): boolean { - return SPACE_SCHEMA_RE.test(name); + return new RegExp(`^${spacePrefix()}[a-z0-9]{12}$`).test(name); } export function isValidSlug(slug: string): boolean { @@ -19,9 +36,9 @@ export function isValidSlug(slug: string): boolean { } export function slugToSchema(slug: string): string { - return `me_${slug}`; + return `${spacePrefix()}${slug}`; } export function schemaToSlug(schema: string): string { - return schema.slice(3); + return schema.slice(spacePrefix().length); } diff --git a/packages/protocol/fields.ts b/packages/protocol/fields.ts index bb68d1d..330c10d 100644 --- a/packages/protocol/fields.ts +++ b/packages/protocol/fields.ts @@ -26,18 +26,29 @@ export const timestampSchema = z.iso.datetime({ offset: true }); // ============================================================================= /** - * ltree path pattern (alphanumeric and underscores, dot-separated). + * User-facing tree-path input pattern. This is the *lenient* wire form, not the + * canonical ltree: separators may be `.` or `/`, a leading `~` is the home + * shortcut, and labels are ltree labels (`[A-Za-z0-9_-]`). The empty string is + * the root. Every handler that accepts this normalizes it server-side via + * `normalizeTreePath` (see packages/database/space/path.ts), which is the + * authoritative validator — it rejects malformed labels and a misplaced `~` + * with a TreePathError mapped to a validation error. This regex is only a cheap + * shape gate so obviously-bad characters (spaces, etc.) fail fast. + * + * Keeping this lenient (rather than the strict canonical ltree) is required so + * the documented `~`/`share` conventions and slash separators actually work + * over the wire on create/update/move/tree/grant — all of which normalize. */ -const ltreePattern = /^([A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*)?$/; +const treePathInputPattern = /^[A-Za-z0-9_~./-]*$/; /** - * Tree path schema (ltree format, allows empty string for root). + * Tree path schema (lenient user-facing input; allows empty string for root). */ export const treePathSchema = z .string() .regex( - ltreePattern, - "must be a valid ltree path (alphanumeric/underscore, dot-separated)", + treePathInputPattern, + "must be a tree path (labels [A-Za-z0-9_-], '.' or '/' separated, optional leading '~')", ); /** diff --git a/packages/server/index.ts b/packages/server/index.ts index af16023..86a5f7c 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -1,26 +1,12 @@ // packages/server/index.ts -import { authStore } from "@memory.build/auth"; -import { - bootstrapSpaceDatabase, - migrateAuth, - migrateCore, - slugToSchema as spaceSlugToSchema, -} from "@memory.build/database"; -import type { EmbeddingConfig } from "@memory.build/embedding"; -import { coreStore } from "@memory.build/engine/core"; -import { - DEFAULT_WORKER_TIMEOUTS, - WorkerPool, - type WorkerTimeouts, -} from "@memory.build/worker"; -import { configure, info, reportError, span } from "@pydantic/logfire-node"; -import postgres from "postgres"; -import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; -import { embeddingConstants } from "./config"; -import type { ServerContext } from "./context"; -import { checkSizeLimit } from "./middleware"; -import { createRouter } from "./router"; -import { internalError } from "./util/response"; +// +// Production entrypoint: configure telemetry, boot the server via startServer(), +// then install the process-level signal/error handlers that index.ts owns (and +// startServer deliberately does not). The actual bootstrap lives in start.ts so +// it can be driven in-process by tests. +import { configure, info, reportError } from "@pydantic/logfire-node"; +import { SERVER_VERSION } from "../../version"; +import { startServer } from "./start"; // Resolve git revision for Logfire code source linking. // Locally, use the actual commit hash for precise source-linking. @@ -57,396 +43,10 @@ configure({ }, }); -// ============================================================================= -// Environment Variables -// ============================================================================= -// -// Required: -// DATABASE_URL - PostgreSQL connection string for the database that -// holds the auth + core control plane and every -// per-space me_ schema (one DB, one pool) -// API_BASE_URL - Public URL for OAuth callbacks -// (e.g., "https://memory.build") -// -// Optional: -// PORT - HTTP server port (default: 3000) -// AUTH_SCHEMA - Auth schema name (default: "auth") -// CORE_SCHEMA - Core control-plane schema name (default: "core") -// -// Connection Pool - Database: -// DB_POOL_MAX - Max connections (default: 20) -// DB_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: 300) -// DB_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: 0) -// DB_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: 30) -// -// Embedding Worker Database: -// WORKER_DATABASE_URL - PostgreSQL connection string for worker traffic (default: DATABASE_URL) -// WORKER_DB_POOL_MAX - Max worker connections (default: WORKER_COUNT) -// WORKER_DB_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: DB_POOL_IDLE_REAP_SECONDS) -// WORKER_DB_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: DB_POOL_MAX_LIFETIME) -// WORKER_DB_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: DB_POOL_CONNECTION_TIMEOUT) -// WORKER_DB_STATEMENT_TIMEOUT - Worker query timeout (default: built-in worker default) -// WORKER_DB_LOCK_TIMEOUT - Worker lock wait timeout (default: built-in worker default) -// WORKER_DB_TRANSACTION_TIMEOUT - Worker transaction timeout (default: built-in worker default) -// WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Worker idle-in-transaction timeout (default: built-in worker default) -// -// Cleanup: -// DEVICE_FLOW_CLEANUP_CRON - Cron schedule for cleaning up expired device auths -// (default: "*/15 * * * *" = every 15 minutes, UTC) -// -// Embedding Worker: -// WORKER_COUNT - Number of concurrent embedding workers (default: 2) -// WORKER_BATCH_SIZE - Queue entries to claim per batch (default: 10) -// WORKER_LOCK_DURATION - PostgreSQL interval for claim lock (default: "5 minutes") -// WORKER_IDLE_DELAY_MS - Poll interval when idle in ms (default: 10000) -// WORKER_MAX_BACKOFF_MS - Max error backoff in ms (default: 60000) -// WORKER_REFRESH_INTERVAL_MS - Space re-discovery interval in ms (default: 60000) -// -// ============================================================================= - -/** - * Parse an integer from an environment variable with NaN guard. - */ -function parseIntEnv( - name: string, - value: string, - defaultValue: string, -): number { - const raw = value || defaultValue; - const parsed = parseInt(raw, 10); - if (Number.isNaN(parsed)) { - throw new Error( - `Invalid value for ${name}: "${raw}" is not a valid integer`, - ); - } - return parsed; -} - -const port = process.env.PORT || 3000; - -const databaseUrl = process.env.DATABASE_URL; -if (!databaseUrl) { - throw new Error("DATABASE_URL environment variable is required"); -} - -const apiBaseUrl = process.env.API_BASE_URL; -if (!apiBaseUrl) { - throw new Error("API_BASE_URL environment variable is required"); -} - -const deviceFlowCleanupCron = - process.env.DEVICE_FLOW_CLEANUP_CRON || "*/15 * * * *"; - -// Schema names (single DB, postgres.js pool): auth + core control plane. -const authSchema = process.env.AUTH_SCHEMA || "auth"; -const coreSchema = process.env.CORE_SCHEMA || "core"; - -const workerCount = parseIntEnv( - "WORKER_COUNT", - process.env.WORKER_COUNT || "", - "2", -); - -// Connection pool settings - database -const dbPoolMax = parseIntEnv( - "DB_POOL_MAX", - process.env.DB_POOL_MAX || "", - "20", -); -const dbPoolIdleReapSeconds = parseIntEnv( - "DB_POOL_IDLE_REAP_SECONDS", - process.env.DB_POOL_IDLE_REAP_SECONDS || "", - "300", -); -const dbPoolMaxLifetime = parseIntEnv( - "DB_POOL_MAX_LIFETIME", - process.env.DB_POOL_MAX_LIFETIME || "", - "0", -); -const dbPoolConnectionTimeout = parseIntEnv( - "DB_POOL_CONNECTION_TIMEOUT", - process.env.DB_POOL_CONNECTION_TIMEOUT || "", - "30", -); - -// Connection pool settings - embedding worker database -const workerDatabaseUrl = process.env.WORKER_DATABASE_URL || databaseUrl; -const workerDbPoolMax = parseIntEnv( - "WORKER_DB_POOL_MAX", - process.env.WORKER_DB_POOL_MAX || "", - String(Math.max(workerCount, 1)), -); -const workerDbPoolIdleReapSeconds = parseIntEnv( - "WORKER_DB_POOL_IDLE_REAP_SECONDS", - process.env.WORKER_DB_POOL_IDLE_REAP_SECONDS || "", - String(dbPoolIdleReapSeconds), -); -const workerDbPoolMaxLifetime = parseIntEnv( - "WORKER_DB_POOL_MAX_LIFETIME", - process.env.WORKER_DB_POOL_MAX_LIFETIME || "", - String(dbPoolMaxLifetime), -); -const workerDbPoolConnectionTimeout = parseIntEnv( - "WORKER_DB_POOL_CONNECTION_TIMEOUT", - process.env.WORKER_DB_POOL_CONNECTION_TIMEOUT || "", - String(dbPoolConnectionTimeout), -); -const workerTimeouts: WorkerTimeouts = { - statementTimeout: - process.env.WORKER_DB_STATEMENT_TIMEOUT ?? - DEFAULT_WORKER_TIMEOUTS.statementTimeout, - lockTimeout: - process.env.WORKER_DB_LOCK_TIMEOUT ?? DEFAULT_WORKER_TIMEOUTS.lockTimeout, - transactionTimeout: - process.env.WORKER_DB_TRANSACTION_TIMEOUT ?? - DEFAULT_WORKER_TIMEOUTS.transactionTimeout, - idleInTransactionSessionTimeout: - process.env.WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? - DEFAULT_WORKER_TIMEOUTS.idleInTransactionSessionTimeout, -}; - -// ============================================================================= -// Embedding Config -// ============================================================================= -// -// Model and dimensions are hardcoded - all spaces use the same embedding model. -// Only the API key is configurable via environment variable. -// -// Required: -// EMBEDDING_API_KEY - OpenAI API key -// -// Optional: -// EMBEDDING_BASE_URL - API base URL (default: OpenAI) -// EMBEDDING_TIMEOUT_MS - Per-call timeout in ms (default: none) -// EMBEDDING_MAX_RETRIES - Retries on transient failures (default: 2, from AI SDK) -// EMBEDDING_MAX_PARALLEL_CALLS - Max concurrent batch chunk requests (default: Infinity) -// -// ============================================================================= - -function buildEmbeddingConfig(): EmbeddingConfig { - const apiKey = process.env.EMBEDDING_API_KEY; - if (!apiKey) { - throw new Error("EMBEDDING_API_KEY is required"); - } - - const options: EmbeddingConfig["options"] = {}; - - if (process.env.EMBEDDING_TIMEOUT_MS) { - options.timeoutMs = parseIntEnv( - "EMBEDDING_TIMEOUT_MS", - process.env.EMBEDDING_TIMEOUT_MS, - "0", - ); - } - if (process.env.EMBEDDING_MAX_RETRIES) { - options.maxRetries = parseIntEnv( - "EMBEDDING_MAX_RETRIES", - process.env.EMBEDDING_MAX_RETRIES, - "0", - ); - } - if (process.env.EMBEDDING_MAX_PARALLEL_CALLS) { - options.maxParallelCalls = parseIntEnv( - "EMBEDDING_MAX_PARALLEL_CALLS", - process.env.EMBEDDING_MAX_PARALLEL_CALLS, - "0", - ); - } - - return { - provider: "openai", - model: embeddingConstants.model, - dimensions: embeddingConstants.dimensions, - apiKey, - baseUrl: process.env.EMBEDDING_BASE_URL, - options, - }; -} - -const embeddingConfig = buildEmbeddingConfig(); - -// ============================================================================= -// OAuth Provider Validation -// ============================================================================= - -// Warn at startup if OAuth providers are not configured, rather than -// failing with a confusing error when someone tries to log in. -const configuredProviders: string[] = []; -if (process.env.GITHUB_CLIENT_ID && process.env.GITHUB_CLIENT_SECRET) { - configuredProviders.push("github"); -} -if (process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) { - configuredProviders.push("google"); -} -if (configuredProviders.length === 0) { - console.warn( - "WARNING: No OAuth providers configured. Set GITHUB_CLIENT_ID/GITHUB_CLIENT_SECRET or GOOGLE_CLIENT_ID/GOOGLE_CLIENT_SECRET.", - ); -} else { - info("OAuth providers configured", { providers: configuredProviders }); -} - -// ============================================================================= -// Database Pools -// ============================================================================= - -// Dedicated worker pool (postgres.js) — the embedding worker processes the -// per-space me_ schemas. -const workerDb = postgres(workerDatabaseUrl, { - max: workerDbPoolMax, - idle_timeout: workerDbPoolIdleReapSeconds, - max_lifetime: workerDbPoolMaxLifetime, - connect_timeout: workerDbPoolConnectionTimeout, - onnotice: () => {}, -}); - -// The single application pool (postgres.js): the auth + core control plane and -// the per-space me_ data schemas all live in one database, one pool. -const db = postgres(databaseUrl, { - max: dbPoolMax, - idle_timeout: dbPoolIdleReapSeconds, - max_lifetime: dbPoolMaxLifetime, - connect_timeout: dbPoolConnectionTimeout, - onnotice: () => {}, -}); - -// Auth store (auth schema) on the application pool. -const auth = authStore(db, authSchema); - -// Core control-plane store (core schema) on the same pool. -const core = coreStore(db, coreSchema); - -// ============================================================================= -// Database Bootstrap & Migrations (blocking — server won't serve until current) -// ============================================================================= - -// Prepare the database for per-space schemas (extensions + roles, idempotent) -// and migrate the auth + core control-plane schemas on the application pool. -// If the DB user lacks CREATE EXTENSION privileges (e.g., RDS), bootstrap -// throws with a clear error describing what's missing. -await bootstrapSpaceDatabase(db); -await migrateCore(db); -await migrateAuth(db); -info("Core + auth schemas migrated"); - -// ============================================================================= -// Router -// ============================================================================= - -const serverContext: ServerContext = { - db, - auth, - core, - authSchema, - coreSchema, - embeddingConfig, - apiBaseUrl, - serverVersion: SERVER_VERSION, - minClientVersion: MIN_CLIENT_VERSION, -}; - -const router = createRouter(serverContext); - -// ============================================================================= -// Embedding Worker Pool -// ============================================================================= - -const workerPool = new WorkerPool(workerDb, { - embedding: embeddingConfig, - discover: async () => { - const spaces = await core.listSpaces(); - return spaces.map((s) => ({ schema: spaceSlugToSchema(s.slug) })); - }, - batchSize: parseIntEnv( - "WORKER_BATCH_SIZE", - process.env.WORKER_BATCH_SIZE || "", - "10", - ), - lockDuration: process.env.WORKER_LOCK_DURATION || "5 minutes", - idleDelayMs: parseIntEnv( - "WORKER_IDLE_DELAY_MS", - process.env.WORKER_IDLE_DELAY_MS || "", - "10000", - ), - maxBackoffMs: parseIntEnv( - "WORKER_MAX_BACKOFF_MS", - process.env.WORKER_MAX_BACKOFF_MS || "", - "60000", - ), - refreshIntervalMs: parseIntEnv( - "WORKER_REFRESH_INTERVAL_MS", - process.env.WORKER_REFRESH_INTERVAL_MS || "", - "60000", - ), - timeouts: workerTimeouts, -}); - -await workerPool.start(workerCount); -info("Embedding worker pool started", { workers: workerCount }); - -// ============================================================================= -// Cleanup Jobs -// ============================================================================= - -// Sweep expired device authorizations and sessions on a cron schedule (UTC). -// Both live in the auth schema now; terminal device states delete themselves on -// poll, so this only reclaims rows that were abandoned before completing. -const cleanupCron = Bun.cron(deviceFlowCleanupCron, async () => { - try { - const devices = await auth.deleteExpiredDevices(); - if (devices > 0) { - info("Cleaned up expired device authorizations", { count: devices }); - } - } catch (error) { - reportError("Failed to cleanup device authorizations", error as Error); - } - try { - const sessions = await auth.cleanupExpiredSessions(); - if (sessions > 0) { - info("Cleaned up expired sessions", { count: sessions }); - } - } catch (error) { - reportError("Failed to cleanup expired sessions", error as Error); - } -}); - -// ============================================================================= -// Server -// ============================================================================= - -const server = Bun.serve({ - port, - async fetch(request) { - const url = new URL(request.url); - const method = request.method; - const path = url.pathname; - - try { - return await span("http.request", { - attributes: { - "http.method": method, - "http.url": request.url, - "http.path": path, - }, - callback: async () => { - // Check size limit - const sizeError = checkSizeLimit(request); - if (sizeError) { - return sizeError; - } - - // Route and handle request - return await router.handleRequest(request); - }, - }); - } catch { - // Error already recorded on http.request span by the helper - return internalError(); - } - }, -}); - -info("Server started", { port }); +// Boot the full stack. All env parsing / pools / migrate / worker / Bun.serve +// happen inside startServer(); see start.ts for the documented environment +// variables. +const srv = await startServer(); // ============================================================================= // Graceful Shutdown @@ -457,32 +57,11 @@ let shutdownRequested = false; async function shutdown() { if (shutdownRequested) return; shutdownRequested = true; - - info("Shutting down server..."); - - // Stop accepting new connections - server.stop(); - - // Stop embedding workers try { - await workerPool.stop(); - info("Embedding worker pool stopped"); + await srv.stop(); } catch (error) { - reportError("Error stopping embedding workers", error as Error); + reportError("Error during shutdown", error as Error); } - - // Stop background jobs - cleanupCron.stop(); - - // Close database pools - try { - await workerDb.end(); - await db.end(); - } catch (error) { - reportError("Error closing database connections", error as Error); - } - - info("Shutdown complete"); process.exit(0); } @@ -501,3 +80,5 @@ process.on("uncaughtException", (error) => { reportError("Uncaught exception", error); process.exit(1); }); + +info("Server ready", { url: srv.url }); diff --git a/packages/server/lib.ts b/packages/server/lib.ts index cb5666c..09142bf 100644 --- a/packages/server/lib.ts +++ b/packages/server/lib.ts @@ -4,3 +4,9 @@ export type { ServerContext } from "./context"; export { extractBearerToken } from "./middleware"; export { createRouter, type Router } from "./router"; +export { + buildEmbeddingConfig, + type RunningServer, + type StartServerOptions, + startServer, +} from "./start"; diff --git a/packages/server/start.integration.test.ts b/packages/server/start.integration.test.ts new file mode 100644 index 0000000..d2924ba --- /dev/null +++ b/packages/server/start.integration.test.ts @@ -0,0 +1,88 @@ +// Integration test for the extracted startServer() bootstrap. +// +// Boots the real server stack (pools → migrate → worker → Bun.serve) against +// isolated auth/core test schemas on a port-0 listener, then hits /health and +// /ready. No real embeddings are exercised (a placeholder key suffices — the +// worker idles with no spaces), so this needs no OpenAI key. +// TEST_DATABASE_URL="$(ghost connect testing_me)" \ +// bun test --timeout 30000 packages/server/start.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import type { EmbeddingConfig } from "@memory.build/embedding"; +import postgres, { type Sql } from "postgres"; +import { startServer } from "./lib"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +const embeddingConfig: EmbeddingConfig = { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + apiKey: "test-key-not-used", + options: {}, +}; + +let sql: Sql; +let srv: Awaited>; +let authSchema: string; +let coreSchema: string; + +beforeAll(async () => { + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + srv = await startServer({ + port: 0, + databaseUrl: URL, + apiBaseUrl: "http://localhost", + authSchema, + coreSchema, + embeddingConfig, + workerCount: 1, + workerIdleDelayMs: 250, + workerRefreshIntervalMs: 500, + enableCleanupCron: false, + // migrate defaults to true — startServer migrates the isolated schemas. + }); + sql = postgres(URL, { onnotice: () => {} }); +}); + +afterAll(async () => { + await srv?.stop(); + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +test("boots on a random port and serves /health", async () => { + expect(srv.port).toBeGreaterThan(0); + const res = await fetch(`${srv.url}/health`); + expect(res.status).toBe(200); + expect(await res.text()).toBe("ok"); +}); + +test("/ready reports the database is reachable", async () => { + const res = await fetch(`${srv.url}/ready`); + expect(res.status).toBe(200); +}); + +test("migrated the configured isolated schemas", async () => { + const [authRow] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${authSchema} + ) as e`; + const [coreRow] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${coreSchema} + ) as e`; + expect(Boolean(authRow?.e)).toBe(true); + expect(Boolean(coreRow?.e)).toBe(true); +}); diff --git a/packages/server/start.ts b/packages/server/start.ts new file mode 100644 index 0000000..b53f1ec --- /dev/null +++ b/packages/server/start.ts @@ -0,0 +1,453 @@ +// packages/server/start.ts +// +// Callable server bootstrap. `startServer()` stands up the same stack the +// production entrypoint (index.ts) runs — pools → bootstrap/migrate → router → +// worker pool → Bun.serve — but with **no** process-level side effects +// (no SIGINT/SIGTERM/unhandledRejection handlers, no process.exit, no +// telemetry configure()). It returns a `RunningServer` handle whose `stop()` +// tears everything down. index.ts is the thin entrypoint that wraps this with +// telemetry + signal handling; the e2e harness calls it directly. +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, + slugToSchema as spaceSlugToSchema, +} from "@memory.build/database"; +import type { EmbeddingConfig } from "@memory.build/embedding"; +import { coreStore } from "@memory.build/engine/core"; +import { + DEFAULT_WORKER_TIMEOUTS, + WorkerPool, + type WorkerTimeouts, +} from "@memory.build/worker"; +import { info, reportError, span } from "@pydantic/logfire-node"; +import postgres from "postgres"; +import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; +import { embeddingConstants } from "./config"; +import type { ServerContext } from "./context"; +import { checkSizeLimit } from "./middleware"; +import { createRouter } from "./router"; +import { internalError } from "./util/response"; + +/** + * Parse an integer from an environment variable with NaN guard. + */ +function parseIntEnv( + name: string, + value: string, + defaultValue: string, +): number { + const raw = value || defaultValue; + const parsed = parseInt(raw, 10); + if (Number.isNaN(parsed)) { + throw new Error( + `Invalid value for ${name}: "${raw}" is not a valid integer`, + ); + } + return parsed; +} + +/** + * Build the embedding config from environment variables. Requires + * EMBEDDING_API_KEY (the server won't boot without it). Used as the default + * when `StartServerOptions.embeddingConfig` is not supplied. + */ +export function buildEmbeddingConfig(): EmbeddingConfig { + const apiKey = process.env.EMBEDDING_API_KEY; + if (!apiKey) { + throw new Error("EMBEDDING_API_KEY is required"); + } + + const options: EmbeddingConfig["options"] = {}; + + if (process.env.EMBEDDING_TIMEOUT_MS) { + options.timeoutMs = parseIntEnv( + "EMBEDDING_TIMEOUT_MS", + process.env.EMBEDDING_TIMEOUT_MS, + "0", + ); + } + if (process.env.EMBEDDING_MAX_RETRIES) { + options.maxRetries = parseIntEnv( + "EMBEDDING_MAX_RETRIES", + process.env.EMBEDDING_MAX_RETRIES, + "0", + ); + } + if (process.env.EMBEDDING_MAX_PARALLEL_CALLS) { + options.maxParallelCalls = parseIntEnv( + "EMBEDDING_MAX_PARALLEL_CALLS", + process.env.EMBEDDING_MAX_PARALLEL_CALLS, + "0", + ); + } + + return { + provider: "openai", + model: embeddingConstants.model, + dimensions: embeddingConstants.dimensions, + apiKey, + baseUrl: process.env.EMBEDDING_BASE_URL, + options, + }; +} + +export interface StartServerOptions { + /** HTTP port. Default PORT env or 3000; 0 = OS-assigned random port. */ + port?: number; + /** Application pool connection string. Default DATABASE_URL. */ + databaseUrl?: string; + /** Worker pool connection string. Default WORKER_DATABASE_URL ?? databaseUrl. */ + workerDatabaseUrl?: string; + /** Public URL for OAuth callbacks. Default API_BASE_URL. */ + apiBaseUrl?: string; + /** Auth schema name. Default AUTH_SCHEMA ?? "auth". */ + authSchema?: string; + /** Core control-plane schema name. Default CORE_SCHEMA ?? "core". */ + coreSchema?: string; + /** Embedding config. Default buildEmbeddingConfig() (reads env). */ + embeddingConfig?: EmbeddingConfig; + /** Number of concurrent embedding workers. Default WORKER_COUNT ?? 2. */ + workerCount?: number; + /** Worker idle poll interval in ms. Default WORKER_IDLE_DELAY_MS ?? 10000. */ + workerIdleDelayMs?: number; + /** Worker space-rediscovery interval in ms. Default WORKER_REFRESH_INTERVAL_MS ?? 60000. */ + workerRefreshIntervalMs?: number; + /** Run the device-flow/session cleanup cron. Default true; harness sets false. */ + enableCleanupCron?: boolean; + /** Run bootstrap + migrate on boot. Default true. */ + migrate?: boolean; +} + +export interface RunningServer { + /** e.g. http://localhost: */ + url: string; + port: number; + context: ServerContext; + /** Tear down: workerPool.stop → cron.stop → server.stop → pools.end. */ + stop(): Promise; +} + +/** + * Boot the server stack and return a handle. No process-level side effects — + * the caller owns signal handling and process exit (index.ts does this). + */ +export async function startServer( + opts: StartServerOptions = {}, +): Promise { + const port = + opts.port ?? (process.env.PORT ? Number(process.env.PORT) : 3000); + + const databaseUrl = opts.databaseUrl ?? process.env.DATABASE_URL; + if (!databaseUrl) { + throw new Error("DATABASE_URL environment variable is required"); + } + + const apiBaseUrl = opts.apiBaseUrl ?? process.env.API_BASE_URL; + if (!apiBaseUrl) { + throw new Error("API_BASE_URL environment variable is required"); + } + + const deviceFlowCleanupCron = + process.env.DEVICE_FLOW_CLEANUP_CRON || "*/15 * * * *"; + + // Schema names (single DB, postgres.js pool): auth + core control plane. + const authSchema = opts.authSchema ?? process.env.AUTH_SCHEMA ?? "auth"; + const coreSchema = opts.coreSchema ?? process.env.CORE_SCHEMA ?? "core"; + + const workerCount = + opts.workerCount ?? + parseIntEnv("WORKER_COUNT", process.env.WORKER_COUNT || "", "2"); + + // Connection pool settings - database + const dbPoolMax = parseIntEnv( + "DB_POOL_MAX", + process.env.DB_POOL_MAX || "", + "20", + ); + const dbPoolIdleReapSeconds = parseIntEnv( + "DB_POOL_IDLE_REAP_SECONDS", + process.env.DB_POOL_IDLE_REAP_SECONDS || "", + "300", + ); + const dbPoolMaxLifetime = parseIntEnv( + "DB_POOL_MAX_LIFETIME", + process.env.DB_POOL_MAX_LIFETIME || "", + "0", + ); + const dbPoolConnectionTimeout = parseIntEnv( + "DB_POOL_CONNECTION_TIMEOUT", + process.env.DB_POOL_CONNECTION_TIMEOUT || "", + "30", + ); + + // Connection pool settings - embedding worker database + const workerDatabaseUrl = + opts.workerDatabaseUrl ?? process.env.WORKER_DATABASE_URL ?? databaseUrl; + const workerDbPoolMax = parseIntEnv( + "WORKER_DB_POOL_MAX", + process.env.WORKER_DB_POOL_MAX || "", + String(Math.max(workerCount, 1)), + ); + const workerDbPoolIdleReapSeconds = parseIntEnv( + "WORKER_DB_POOL_IDLE_REAP_SECONDS", + process.env.WORKER_DB_POOL_IDLE_REAP_SECONDS || "", + String(dbPoolIdleReapSeconds), + ); + const workerDbPoolMaxLifetime = parseIntEnv( + "WORKER_DB_POOL_MAX_LIFETIME", + process.env.WORKER_DB_POOL_MAX_LIFETIME || "", + String(dbPoolMaxLifetime), + ); + const workerDbPoolConnectionTimeout = parseIntEnv( + "WORKER_DB_POOL_CONNECTION_TIMEOUT", + process.env.WORKER_DB_POOL_CONNECTION_TIMEOUT || "", + String(dbPoolConnectionTimeout), + ); + const workerTimeouts: WorkerTimeouts = { + statementTimeout: + process.env.WORKER_DB_STATEMENT_TIMEOUT ?? + DEFAULT_WORKER_TIMEOUTS.statementTimeout, + lockTimeout: + process.env.WORKER_DB_LOCK_TIMEOUT ?? DEFAULT_WORKER_TIMEOUTS.lockTimeout, + transactionTimeout: + process.env.WORKER_DB_TRANSACTION_TIMEOUT ?? + DEFAULT_WORKER_TIMEOUTS.transactionTimeout, + idleInTransactionSessionTimeout: + process.env.WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? + DEFAULT_WORKER_TIMEOUTS.idleInTransactionSessionTimeout, + }; + + const embeddingConfig = opts.embeddingConfig ?? buildEmbeddingConfig(); + + // OAuth provider validation: warn at startup if none configured, rather than + // failing with a confusing error when someone tries to log in. + const configuredProviders: string[] = []; + if (process.env.GITHUB_CLIENT_ID && process.env.GITHUB_CLIENT_SECRET) { + configuredProviders.push("github"); + } + if (process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) { + configuredProviders.push("google"); + } + if (configuredProviders.length === 0) { + console.warn( + "WARNING: No OAuth providers configured. Set GITHUB_CLIENT_ID/GITHUB_CLIENT_SECRET or GOOGLE_CLIENT_ID/GOOGLE_CLIENT_SECRET.", + ); + } else { + info("OAuth providers configured", { providers: configuredProviders }); + } + + // --------------------------------------------------------------------------- + // Database Pools + // --------------------------------------------------------------------------- + + // Dedicated worker pool (postgres.js) — the embedding worker processes the + // per-space me_ schemas. + const workerDb = postgres(workerDatabaseUrl, { + max: workerDbPoolMax, + idle_timeout: workerDbPoolIdleReapSeconds, + max_lifetime: workerDbPoolMaxLifetime, + connect_timeout: workerDbPoolConnectionTimeout, + onnotice: () => {}, + }); + + // The single application pool (postgres.js): the auth + core control plane and + // the per-space me_ data schemas all live in one database, one pool. + const db = postgres(databaseUrl, { + max: dbPoolMax, + idle_timeout: dbPoolIdleReapSeconds, + max_lifetime: dbPoolMaxLifetime, + connect_timeout: dbPoolConnectionTimeout, + onnotice: () => {}, + }); + + // Auth store (auth schema) on the application pool. + const auth = authStore(db, authSchema); + + // Core control-plane store (core schema) on the same pool. + const core = coreStore(db, coreSchema); + + // --------------------------------------------------------------------------- + // Database Bootstrap & Migrations (blocking — server won't serve until current) + // --------------------------------------------------------------------------- + + // Prepare the database for per-space schemas (extensions + roles, idempotent) + // and migrate the auth + core control-plane schemas on the application pool. + // Pass the configured schemas to BOTH the migrations and the stores so an + // isolated (non-default) schema is migrated where the stores read it. + if (opts.migrate ?? true) { + await bootstrapSpaceDatabase(db); + await migrateCore(db, { schema: coreSchema }); + await migrateAuth(db, { schema: authSchema }); + info("Core + auth schemas migrated"); + } + + // --------------------------------------------------------------------------- + // Router + // --------------------------------------------------------------------------- + + const context: ServerContext = { + db, + auth, + core, + authSchema, + coreSchema, + embeddingConfig, + apiBaseUrl, + serverVersion: SERVER_VERSION, + minClientVersion: MIN_CLIENT_VERSION, + }; + + const router = createRouter(context); + + // --------------------------------------------------------------------------- + // Embedding Worker Pool + // --------------------------------------------------------------------------- + + const workerPool = new WorkerPool(workerDb, { + embedding: embeddingConfig, + discover: async () => { + const spaces = await core.listSpaces(); + return spaces.map((s) => ({ schema: spaceSlugToSchema(s.slug) })); + }, + batchSize: parseIntEnv( + "WORKER_BATCH_SIZE", + process.env.WORKER_BATCH_SIZE || "", + "10", + ), + lockDuration: process.env.WORKER_LOCK_DURATION || "5 minutes", + idleDelayMs: + opts.workerIdleDelayMs ?? + parseIntEnv( + "WORKER_IDLE_DELAY_MS", + process.env.WORKER_IDLE_DELAY_MS || "", + "10000", + ), + maxBackoffMs: parseIntEnv( + "WORKER_MAX_BACKOFF_MS", + process.env.WORKER_MAX_BACKOFF_MS || "", + "60000", + ), + refreshIntervalMs: + opts.workerRefreshIntervalMs ?? + parseIntEnv( + "WORKER_REFRESH_INTERVAL_MS", + process.env.WORKER_REFRESH_INTERVAL_MS || "", + "60000", + ), + timeouts: workerTimeouts, + }); + + await workerPool.start(workerCount); + info("Embedding worker pool started", { workers: workerCount }); + + // --------------------------------------------------------------------------- + // Cleanup Jobs + // --------------------------------------------------------------------------- + + // Sweep expired device authorizations and sessions on a cron schedule (UTC). + // Both live in the auth schema now; terminal device states delete themselves + // on poll, so this only reclaims rows abandoned before completing. + const cleanupCron = + (opts.enableCleanupCron ?? true) + ? Bun.cron(deviceFlowCleanupCron, async () => { + try { + const devices = await auth.deleteExpiredDevices(); + if (devices > 0) { + info("Cleaned up expired device authorizations", { + count: devices, + }); + } + } catch (error) { + reportError( + "Failed to cleanup device authorizations", + error as Error, + ); + } + try { + const sessions = await auth.cleanupExpiredSessions(); + if (sessions > 0) { + info("Cleaned up expired sessions", { count: sessions }); + } + } catch (error) { + reportError("Failed to cleanup expired sessions", error as Error); + } + }) + : null; + + // --------------------------------------------------------------------------- + // Server + // --------------------------------------------------------------------------- + + const server = Bun.serve({ + port, + async fetch(request) { + const url = new URL(request.url); + const method = request.method; + const path = url.pathname; + + try { + return await span("http.request", { + attributes: { + "http.method": method, + "http.url": request.url, + "http.path": path, + }, + callback: async () => { + const sizeError = checkSizeLimit(request); + if (sizeError) { + return sizeError; + } + return await router.handleRequest(request); + }, + }); + } catch { + // Error already recorded on http.request span by the helper + return internalError(); + } + }, + }); + + info("Server started", { port: server.port }); + + let stopped = false; + async function stop(): Promise { + if (stopped) return; + stopped = true; + + info("Shutting down server..."); + + // Stop accepting new connections + server.stop(); + + // Stop embedding workers + try { + await workerPool.stop(); + info("Embedding worker pool stopped"); + } catch (error) { + reportError("Error stopping embedding workers", error as Error); + } + + // Stop background jobs + cleanupCron?.stop(); + + // Close database pools + try { + await workerDb.end(); + await db.end(); + } catch (error) { + reportError("Error closing database connections", error as Error); + } + + info("Shutdown complete"); + } + + const boundPort = server.port ?? port; + return { + url: `http://localhost:${boundPort}`, + port: boundPort, + context, + stop, + }; +} From a02e6b2ba599a7b004443525fc67534ebda7e579 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 8 Jun 2026 12:47:50 +0200 Subject: [PATCH 103/156] feat(cli): MCP install defaults to your login session, not an api key `me install` (codex/gemini/opencode/claude) previously required an api key and baked it literally into the MCP config. Make the default path use your `me login` session instead: - buildMeCommand bakes only `--server` by default; `me mcp` resolves the session token from the keychain/config at runtime (so it survives re-login) and the space from ME_SPACE/active space at runtime. Nothing secret or stateful is written into the config. - `--api-key`/ME_API_KEY switches to the headless path: bakes the global key and requires a pinned `--space`. - `--space` pins the space on either path; otherwise it resolves at runtime. Install now requires being logged in (or an explicit api key) and warns if no active space is set. Update the four tool install docs and option help text. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/cli/me-claude.md | 5 ++- docs/cli/me-codex.md | 5 ++- docs/cli/me-gemini.md | 5 ++- docs/cli/me-opencode.md | 5 ++- packages/cli/commands/claude.ts | 7 +++- packages/cli/commands/codex.ts | 7 +++- packages/cli/commands/gemini.ts | 7 +++- packages/cli/commands/opencode.ts | 7 +++- packages/cli/mcp/agent-install.ts | 62 ++++++++++++++++++------------- packages/cli/mcp/install.test.ts | 40 +++++++++++++++----- packages/cli/mcp/install.ts | 36 +++++++++--------- 11 files changed, 118 insertions(+), 68 deletions(-) diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index cbd5b2a..78b8b37 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -22,11 +22,12 @@ me claude install [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | +| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | | `--server ` | Server URL to embed in the MCP config. | | `-s, --scope ` | Claude Code config scope: `local`, `user`, or `project`. Default: `user`. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. The `--scope` flag mirrors `claude mcp add --scope`: diff --git a/docs/cli/me-codex.md b/docs/cli/me-codex.md index febc4b1..24ad505 100644 --- a/docs/cli/me-codex.md +++ b/docs/cli/me-codex.md @@ -19,10 +19,11 @@ me codex install [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | +| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | | `--server ` | Server URL to embed in the MCP config. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). diff --git a/docs/cli/me-gemini.md b/docs/cli/me-gemini.md index bd6a267..361990a 100644 --- a/docs/cli/me-gemini.md +++ b/docs/cli/me-gemini.md @@ -18,10 +18,11 @@ me gemini install [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | +| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | | `--server ` | Server URL to embed in the MCP config. | | `-s, --scope ` | Gemini CLI config scope: `user` or `project`. Default: `user`. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). diff --git a/docs/cli/me-opencode.md b/docs/cli/me-opencode.md index 12919e3..b9fb2df 100644 --- a/docs/cli/me-opencode.md +++ b/docs/cli/me-opencode.md @@ -19,10 +19,11 @@ me opencode install [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | +| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | | `--server ` | Server URL to embed in the MCP config. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 25f1993..077aff3 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -54,11 +54,14 @@ function parseClaudeScope(value: string): ClaudeScope { function createClaudeInstallCommand(): Command { return new Command("install") .description("register me as an MCP server with Claude Code") - .option("--api-key ", "API key to embed in MCP config") + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) .option("--server ", "server URL to embed in MCP config") .option( "--space ", - "space to embed in MCP config (else ME_SPACE / active space)", + "pin a space (default: resolve ME_SPACE / active space at runtime)", ) .option( "-s, --scope ", diff --git a/packages/cli/commands/codex.ts b/packages/cli/commands/codex.ts index dc3c4af..ba80ea5 100644 --- a/packages/cli/commands/codex.ts +++ b/packages/cli/commands/codex.ts @@ -14,11 +14,14 @@ import { buildAgentImportSubcommand } from "./import.ts"; function createCodexInstallCommand(): Command { return new Command("install") .description("register me as an MCP server with Codex CLI") - .option("--api-key ", "API key to embed in MCP config") + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) .option("--server ", "server URL to embed in MCP config") .option( "--space ", - "space to embed in MCP config (else ME_SPACE / active space)", + "pin a space (default: resolve ME_SPACE / active space at runtime)", ) .action(async (opts: AgentInstallOptions, cmd: Command) => { const globalOpts = cmd.optsWithGlobals(); diff --git a/packages/cli/commands/gemini.ts b/packages/cli/commands/gemini.ts index d16d938..c76b302 100644 --- a/packages/cli/commands/gemini.ts +++ b/packages/cli/commands/gemini.ts @@ -24,11 +24,14 @@ function parseGeminiScope(value: string): GeminiScope { function createGeminiInstallCommand(): Command { return new Command("install") .description("register me as an MCP server with Gemini CLI") - .option("--api-key ", "API key to embed in MCP config") + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) .option("--server ", "server URL to embed in MCP config") .option( "--space ", - "space to embed in MCP config (else ME_SPACE / active space)", + "pin a space (default: resolve ME_SPACE / active space at runtime)", ) .option( "-s, --scope ", diff --git a/packages/cli/commands/opencode.ts b/packages/cli/commands/opencode.ts index 849a8c5..9989576 100644 --- a/packages/cli/commands/opencode.ts +++ b/packages/cli/commands/opencode.ts @@ -14,11 +14,14 @@ import { buildAgentImportSubcommand } from "./import.ts"; function createOpenCodeInstallCommand(): Command { return new Command("install") .description("register me as an MCP server with OpenCode") - .option("--api-key ", "API key to embed in MCP config") + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) .option("--server ", "server URL to embed in MCP config") .option( "--space ", - "space to embed in MCP config (else ME_SPACE / active space)", + "pin a space (default: resolve ME_SPACE / active space at runtime)", ) .action(async (opts: AgentInstallOptions, cmd: Command) => { const globalOpts = cmd.optsWithGlobals(); diff --git a/packages/cli/mcp/agent-install.ts b/packages/cli/mcp/agent-install.ts index 4bae7f6..5f5fe69 100644 --- a/packages/cli/mcp/agent-install.ts +++ b/packages/cli/mcp/agent-install.ts @@ -36,35 +36,48 @@ export async function runAgentMcpInstall( process.exit(1); } - // Resolve credentials: flags > env (ME_API_KEY / ME_SPACE) > stored config. - // MCP configs bake in a long-lived agent api key (a human session would - // expire), so an api key is required here — mint one with - // `me apikey create `. Keys are global, so a space must be baked in too. - let { apiKey, server, space } = opts; - if (!apiKey || !server || !space) { - const creds = resolveCredentials(server); - if (!apiKey) apiKey = creds.apiKey; - if (!server) server = creds.server; - if (!space) space = creds.activeSpace; - } - - if (!apiKey) { - clack.log.error( - "No API key available. Pass --api-key or set ME_API_KEY — mint one with 'me apikey create '.", - ); - process.exit(1); - } + // Resolve credentials: flags > env (ME_API_KEY / ME_SERVER / ME_SPACE) > + // stored config. + const creds = resolveCredentials(opts.server); + const apiKey = opts.apiKey ?? creds.apiKey; // --api-key > ME_API_KEY + const server = opts.server ?? creds.server; if (!server) { clack.log.error("No server URL available. Pass --server or set ME_SERVER."); process.exit(1); } - if (!space) { - clack.log.error( - "No space available. Pass --space, set ME_SPACE, or run 'me space use '.", - ); - process.exit(1); + // Default path: no api key → the MCP server uses your login SESSION, resolved + // from the keychain/config at runtime each time it starts (so it survives + // `me login`). Pass --api-key / ME_API_KEY only for a headless agent that + // can't reach your keychain; that bakes a long-lived global key and must pin a + // space. The `--space` flag pins the space either way; otherwise the session + // path resolves it at runtime from ME_SPACE / active space. + let meCmd: string[]; + if (apiKey) { + const space = opts.space ?? creds.activeSpace; + if (!space) { + clack.log.error( + "No space for the API key. Pass --space, set ME_SPACE, or run 'me space use ' (keys are global, so the space must be fixed).", + ); + process.exit(1); + } + meCmd = buildMeCommand({ server, apiKey, space }); + } else { + if (!creds.sessionToken) { + clack.log.error( + "Not logged in. Run 'me login' (the MCP server will use your session), or pass --api-key / set ME_API_KEY for a headless agent.", + ); + process.exit(1); + } + // Bake only --server (+ an explicit --space pin if given); the session token + // and space resolve at runtime. + meCmd = buildMeCommand({ server, space: opts.space }); + if (!opts.space && !creds.activeSpace) { + clack.log.warn( + "No active space set — the MCP server will fail until you run 'me space use ' (or set ME_SPACE). Re-run with --space to pin one.", + ); + } } // For CLI tools, require the binary to be on PATH. JSON-file tools @@ -76,9 +89,6 @@ export async function runAgentMcpInstall( process.exit(1); } - // Build the me mcp command with baked-in credentials - const meCmd = buildMeCommand(apiKey, server, space); - const spin = clack.spinner(); spin.start(`Registering with ${tool.name}...`); const result = await installMcpServer(tool, meCmd, { scope: opts.scope }); diff --git a/packages/cli/mcp/install.test.ts b/packages/cli/mcp/install.test.ts index 1ffe47d..704b9bd 100644 --- a/packages/cli/mcp/install.test.ts +++ b/packages/cli/mcp/install.test.ts @@ -6,28 +6,50 @@ import { buildMeCommand, buildOpenCodeConfig, MCP_TOOLS } from "./install.ts"; describe("buildMeCommand", () => { test("uses bare 'me' command on PATH", () => { - const cmd = buildMeCommand( - "test-key-123", - "https://api.memory.build", - "abc123def456", - ); + const cmd = buildMeCommand({ server: "https://api.memory.build" }); expect(cmd[0]).toBe("me"); expect(cmd[1]).toBe("mcp"); }); - test("includes --api-key, --server, and --space with correct values", () => { - const cmd = buildMeCommand("k", "https://example.com", "abc123def456"); + test("session default bakes only --server (token + space resolve at runtime)", () => { + const cmd = buildMeCommand({ server: "https://example.com" }); + expect(cmd).toEqual(["me", "mcp", "--server", "https://example.com"]); + expect(cmd).not.toContain("--api-key"); + expect(cmd).not.toContain("--space"); + }); + + test("pins --space when given (session path with explicit space)", () => { + const cmd = buildMeCommand({ + server: "https://example.com", + space: "abc123def456", + }); expect(cmd).toEqual([ "me", "mcp", - "--api-key", - "k", "--server", "https://example.com", "--space", "abc123def456", ]); }); + + test("headless agent bakes --api-key and --space", () => { + const cmd = buildMeCommand({ + server: "https://example.com", + apiKey: "k", + space: "abc123def456", + }); + expect(cmd).toEqual([ + "me", + "mcp", + "--server", + "https://example.com", + "--api-key", + "k", + "--space", + "abc123def456", + ]); + }); }); describe("buildOpenCodeConfig", () => { diff --git a/packages/cli/mcp/install.ts b/packages/cli/mcp/install.ts index cf450b0..a0ada7d 100644 --- a/packages/cli/mcp/install.ts +++ b/packages/cli/mcp/install.ts @@ -114,27 +114,29 @@ export function detectInstalledTools(): McpTool[] { } /** - * Build the `me mcp` command array with baked-in credentials. + * Build the `me mcp …` command array to embed in an MCP config. + * + * Only `--server` is always baked. `--api-key` and `--space` are baked **only** + * when provided: + * - **Default (no api key):** the MCP server resolves your login *session* from + * the keychain/config at runtime (so it keeps working across `me login`), and + * the space from `ME_SPACE`/active space at runtime — nothing secret or + * stateful is written into the config. + * - **Headless agent (`--api-key`):** the global key is baked in, along with a + * pinned `--space` (keys aren't space-bound, so the space must be fixed). * - * Api keys are global, so the space must be baked in explicitly (`--space`). * Always uses bare `me` — the binary is expected to be on PATH whether installed * via the install script, Homebrew, or npm. */ -export function buildMeCommand( - apiKey: string, - serverUrl: string, - space: string, -): string[] { - return [ - "me", - "mcp", - "--api-key", - apiKey, - "--server", - serverUrl, - "--space", - space, - ]; +export function buildMeCommand(opts: { + server: string; + apiKey?: string; + space?: string; +}): string[] { + const cmd = ["me", "mcp", "--server", opts.server]; + if (opts.apiKey) cmd.push("--api-key", opts.apiKey); + if (opts.space) cmd.push("--space", opts.space); + return cmd; } // ============================================================================= From a4a2bbd1d053c627be8d0f506c6ddf36ac492a41 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 8 Jun 2026 13:11:29 +0200 Subject: [PATCH 104/156] =?UTF-8?q?feat(auth):=20rolling=20sessions=20(7d,?= =?UTF-8?q?=20daily=20refresh,=20no=20cap)=20=E2=80=94=20better-auth=20mod?= =?UTF-8?q?el?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sessions were a fixed 30-day expiry with no renewal, so an actively-used login died monthly regardless of activity — now user-visible since MCP installs default to the session. Adopt better-auth's rolling model: - validate_session slides expires_at to now + 7d on use, throttled to ~once/day (only when <6d remains = window - updateAge). The function is now volatile and writes at most ~once/session/day on the hot path. No absolute cap. - Initial session window 30d → 7d (SESSION_EXPIRY_DAYS), in sync with the SQL. An active session never expires; an idle one lapses 7d after last use. The no-absolute-cap tradeoff (OWASP would pair an absolute timeout) is recorded in DECISIONS_FOR_REVIEW.md with how to add one later. Co-Authored-By: Claude Opus 4.8 (1M context) --- DECISIONS_FOR_REVIEW.md | 35 +++++++++++++++++++ packages/auth/db.integration.test.ts | 17 +++++++++ packages/auth/db.ts | 6 +++- .../auth/migrate/idempotent/002_session.sql | 35 +++++++++++++++---- 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index 8afe161..2d7d630 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -124,6 +124,41 @@ making them mintable for users widens that surface. --- +## Rolling sessions (7d, refreshed daily, no absolute cap) — copied from better-auth + +**Date:** 2026-06-08 · **Area:** auth / session lifetime + +Sessions were a **fixed** 30-day expiry with no renewal — an actively-used login +died 30 days after `me login` regardless of activity. That became user-visible +once `me install` started defaulting to the login session (a logged-in +editor's MCP integration would silently break monthly). Changed to **rolling** +sessions matching better-auth's defaults: `validate_session` slides `expires_at` +to `now + 7d` on use, **throttled to ~once/day** (only when <6d remains, i.e. +window − updateAge), with **no absolute cap**. The function is now `volatile` +(was `stable`) and does at most ~one write/session/day on the hot path. + +**Decision:** adopt better-auth's model verbatim (expiresIn=7d, updateAge=1d, no +cap). Initial window also dropped 30d → 7d (`SESSION_EXPIRY_DAYS`). + +**The open tradeoff — no absolute cap:** OWASP recommends pairing an idle timeout +(which we now have: 7d) with an **absolute timeout** (a hard ceiling regardless of +activity) so a leaked-but-actively-used session can't roll forever. better-auth +omits this by default and we followed suit, prioritizing "never log out an active +user." A continuously-used (or exfiltrated-and-used) session never expires; +mitigations remain `me logout` / `deleteSessionsByUser` (revoke-all). + +**How to add a cap later:** store `absolute_expires_at = created_at + ` (or +compute from the existing `sessions.created_at`) and `least(now()+7d, +absolute_expires_at)` in the `validate_session` bump; force re-login past it. +Window/throttle live in `packages/database/auth/migrate/idempotent/002_session.sql` +(`validate_session`) and `SESSION_EXPIRY_DAYS` in `packages/auth/db.ts` — keep the +two windows in sync. + +**Status:** decided (copy better-auth); revisit the absolute cap if the +long-lived-bearer surface becomes a concern. + +--- + ## Should an agent get `share` access on join by default, or no grants (as now)? **Date:** 2026-06-08 · **Area:** membership (`me agent add` / `principal.add`) diff --git a/packages/auth/db.integration.test.ts b/packages/auth/db.integration.test.ts index 88ecff1..ff337ec 100644 --- a/packages/auth/db.integration.test.ts +++ b/packages/auth/db.integration.test.ts @@ -70,6 +70,23 @@ test("createSession returns a token that validateSession accepts", async () => { expect(await db.validateSession(token)).toBeNull(); }); +test("validateSession rolls a near-expiry session forward, then throttles", async () => { + const id = await db.createUser(email(), "Rolling"); + // Start with a 1-day session so it's inside the refresh threshold (<6d left). + const { token } = await db.createSession(id, { expiresInDays: 1 }); + + // First use slides the expiry forward to ~7 days out (the window). + const before = await db.validateSession(token); + expect(before).not.toBeNull(); + const sixDaysOut = Date.now() + 6 * 24 * 60 * 60 * 1000; + expect(before?.expiresAt.getTime()).toBeGreaterThan(sixDaysOut); + + // A full-window session is NOT bumped again on the next use (≤ once/day + // throttle): the stored expiry is unchanged, not sliding on every call. + const after = await db.validateSession(token); + expect(after?.expiresAt.getTime()).toBe(before?.expiresAt.getTime()); +}); + test("deleteSessionsByUser revokes all of a user's sessions", async () => { const id = await db.createUser(email(), "Carol"); const a = await db.createSession(id); diff --git a/packages/auth/db.ts b/packages/auth/db.ts index 8bb81c9..248ef1b 100644 --- a/packages/auth/db.ts +++ b/packages/auth/db.ts @@ -22,7 +22,11 @@ import type { ValidatedSession, } from "./types"; -const SESSION_EXPIRY_DAYS = 30; +// Initial session lifetime at login. Sessions are rolling: validate_session +// slides expiry forward to now + this window on use (throttled to ~once/day), +// with no absolute cap — better-auth's model (expiresIn=7d, updateAge=1d). Keep +// this in sync with the window in auth migrate 002_session.sql validate_session. +const SESSION_EXPIRY_DAYS = 7; /** * The auth control-plane data layer. diff --git a/packages/database/auth/migrate/idempotent/002_session.sql b/packages/database/auth/migrate/idempotent/002_session.sql index b61d1a3..b56d750 100644 --- a/packages/database/auth/migrate/idempotent/002_session.sql +++ b/packages/database/auth/migrate/idempotent/002_session.sql @@ -21,6 +21,13 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- validate_session -- Looks up an unexpired session by token hash and returns the session + its -- user. No rows if missing or expired. +-- +-- Rolling session (better-auth model): on a valid lookup the expiry slides +-- forward to now + 7 days, but at most once per day — only when the remaining +-- lifetime has dropped below (window - updateAge) = 6 days. So an actively-used +-- session never expires, an idle one lapses 7 days after last use, and the hot +-- path writes at most ~once/day/session (the function is therefore volatile). +-- No absolute cap, matching better-auth's defaults (expiresIn=7d, updateAge=1d). ------------------------------------------------------------------------------- create or replace function {{schema}}.validate_session(_token_hash bytea) returns table @@ -31,12 +38,28 @@ returns table , expires_at timestamptz ) as $func$ - select s.id, u.id, u.email::text, u.name, s.expires_at - from {{schema}}.sessions s - inner join {{schema}}.users u on (u.id = s.user_id) - where s.token_hash = _token_hash - and s.expires_at > now() -$func$ language sql stable security invoker + with valid as + ( + select s.id, s.user_id, s.expires_at + from {{schema}}.sessions s + where s.token_hash = _token_hash + and s.expires_at > now() + ) + , bumped as + ( + update {{schema}}.sessions s + set expires_at = now() + interval '7 days' -- window (expiresIn) + from valid v + where s.id = v.id + and v.expires_at < now() + interval '6 days' -- throttle: window - updateAge (1d) + returning s.id, s.expires_at + ) + select v.id, u.id, u.email::text, u.name + , coalesce(b.expires_at, v.expires_at) as expires_at + from valid v + inner join {{schema}}.users u on (u.id = v.user_id) + left join bumped b on (b.id = v.id) +$func$ language sql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; From cd349342924f6fc110e5ae1e76d2993409e7d67b Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Mon, 8 Jun 2026 13:29:16 +0200 Subject: [PATCH 105/156] feat(claude-plugin): make api_key optional, fall back to the login session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the Claude Code plugin in line with the new session-default for MCP. The plugin previously hard-required an api key for both its bundled MCP server and its capture hooks; now api_key is optional and both fall back to your `me login` session (captures then attributed to you; set an api key to attribute them to a dedicated agent). - plugin.json: api_key required → optional, with guidance; space stays required. - capture hooks: resolveHookConfigFromEnv takes injected fallback creds and uses plugin api_key > ME_API_KEY > session token; space from plugin config or active space. HookConfig.apiKey renamed to token (it may be either bearer). The hook command passes resolveCredentials() as the fallback. - me mcp: treat a blank/empty or unsubstituted ${user_config.x} flag value as unset so the plugin's static .mcp.json (--api-key ${user_config.api_key}) falls through to the session when api_key is left blank. (Claude Code's substitution for an unset optional userConfig value is undocumented, so guard both forms.) - Update plugin README + me-claude docs (api_key optional, new error text); the README's older engine-era examples are corrected where touched. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/cli/me-claude.md | 4 +- .../claude-plugin/.claude-plugin/plugin.json | 8 +-- packages/claude-plugin/README.md | 45 ++++++------- packages/cli/claude/capture.test.ts | 51 +++++++++++++-- packages/cli/claude/capture.ts | 65 ++++++++++++++----- packages/cli/commands/claude.ts | 25 ++++--- packages/cli/commands/mcp.ts | 17 ++++- 7 files changed, 154 insertions(+), 61 deletions(-) diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index 78b8b37..da9963d 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -57,9 +57,11 @@ This command is not run directly -- the Claude Code plugin calls it. The plugin claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] # then, in a Claude Code session: -/plugin # select memory-engine, Configure, fill api_key/server/tree_prefix +/plugin # select memory-engine, Configure, set space (api_key optional) ``` +`api_key` is optional: leave it blank and the plugin's hooks and MCP server use your `me login` session; set it to attribute captures to a dedicated agent instead. + If you only want the MCP tools (no hooks, no slash commands), use [me claude install](#me-claude-install) instead. Best-effort: logs failures to stderr but always exits 0 so that a hook failure never blocks a Claude Code session. diff --git a/packages/claude-plugin/.claude-plugin/plugin.json b/packages/claude-plugin/.claude-plugin/plugin.json index 7bf50b3..52b1abe 100644 --- a/packages/claude-plugin/.claude-plugin/plugin.json +++ b/packages/claude-plugin/.claude-plugin/plugin.json @@ -12,10 +12,10 @@ "userConfig": { "api_key": { "type": "string", - "title": "API key", - "description": "Memory Engine API key. Create one with `me apikey create` (or have an admin create one with restricted privileges for your agent).", + "title": "API key (optional)", + "description": "Optional. Leave blank to use your `me login` session — the plugin falls back to it automatically. Set an API key to attribute captures to a dedicated agent instead: create one with `me apikey create` (or have an admin create one with restricted privileges).", "sensitive": true, - "required": true + "required": false }, "server": { "type": "string", @@ -27,7 +27,7 @@ "space": { "type": "string", "title": "Space", - "description": "Space slug to store memories in (the X-Me-Space). API keys are global, so the space must be set explicitly.", + "description": "Space slug to store memories in (the X-Me-Space). Required — the active space isn't read from your CLI config here.", "required": true }, "tree_prefix": { diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index 7aad5c5..f481aa2 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -18,35 +18,30 @@ Captures your Claude Code conversations to [Memory Engine](https://memory.build) curl -fsSL https://install.memory.build | sh ``` -2. **Logged in** to a Memory Engine instance with an active engine: +2. **Logged in** to a Memory Engine instance, with an active space selected: ```bash me login - me whoami # confirms identity + active engine + me space use # select the space to capture into + me whoami # confirms identity + active space ``` -3. **An API key** for the plugin. When you `me login`, an admin key for your identity is issued automatically and stored in your local credentials — you can look it up with: + That login session is all the plugin needs — `api_key` is **optional** (see below). - ```bash - # inspect your credentials file (contains the key for the active engine) - cat ~/.config/me/credentials.yaml - ``` - - To paste that key into the plugin is the simplest path. - -### Restricting the plugin's privileges (optional but recommended) +### Using a dedicated agent key (optional) -The key you configure for the plugin does **not** have to be your admin key. You can issue a separate, scoped-down key and paste *that* into the plugin, so the agent only has access to the tree paths you want it to touch: +By default the plugin uses your `me login` session, so captures are attributed to **you**. To attribute captures to a separate, scoped-down **agent** identity instead — so it only touches the tree paths you allow — mint an api key and paste *that* into the plugin's `api_key`: ```bash -# 1. Create a dedicated engine user for the agent -me user create claude-code-agent +# 1. Create a dedicated agent +me agent create claude-code-agent -# 2. Grant it just the access it needs — in this example, read+create on +# 2. Add it to the space and grant just the access it needs — e.g. read+write on # the capture subtree (grants cover all descendant paths via ltree) -me grant create claude-code-agent claude_code.sessions read create +me agent add claude-code-agent +me access grant claude-code-agent claude_code.sessions w -# 3. Issue an API key for that user +# 3. Mint an API key for that agent me apikey create claude-code-agent plugin-key # → prints the raw key once; paste it into the plugin's api_key config ``` @@ -69,20 +64,20 @@ claude plugin install memory-engine@memory-engine --scope local # this repo, ## Configure -The plugin needs four values: `api_key`, `server`, `space`, and `tree_prefix`. Claude Code does not prompt for them at install time — you configure them from inside a session. +The only required value is `space`. Claude Code does not prompt at install time — you configure from inside a session. ```text claude # start a session /plugin # open the plugin manager # → Installed → memory-engine → Configure -# → api_key (sensitive — stored in keychain) +# → space (the space slug — REQUIRED) +# → api_key (OPTIONAL, sensitive — blank = use your `me login` session) # → server (default https://api.memory.build) -# → space (the space slug — api keys are global, so this is required) # → tree_prefix (default claude_code.sessions) # → values take effect immediately; no restart required ``` -Sensitive values (the api_key) go to your system keychain. Non-sensitive values go to the `settings.json` for the scope you installed in. +Leave `api_key` blank to use your `me login` session (captures attributed to you); set it to use a dedicated agent key (see above). Sensitive values (the api_key) go to your system keychain; non-sensitive values go to the `settings.json` for the scope you installed in. ## Verify @@ -140,11 +135,11 @@ Claude Code handles the cleanup. Your captured memories and API keys are preserv ## Troubleshooting -**`[memory-engine] CLAUDE_PLUGIN_OPTION_API_KEY not set` in stderr** -The hook ran but userConfig isn't filled in. Open `/plugin → memory-engine → Configure` and set the api_key. +**`[memory-engine] no credentials` in stderr** +The hook ran but found neither a `me login` session nor a configured api_key. Run `me login` (and `me space use `), or open `/plugin → memory-engine → Configure` and set the api_key + space. **`Plugin option "X" isn't set` in Claude Code's error panel** -A required userConfig value is missing for either a hook or the MCP server. Configure all four: api_key, server, space, tree_prefix. +A required userConfig value is missing — `space` is required (api_key is optional). Configure it via `/plugin → memory-engine → Configure`. **Hook fires but no memories appear** - Confirm the api_key is valid: @@ -155,4 +150,4 @@ A required userConfig value is missing for either a hook or the MCP server. Conf - Confirm `me` is on PATH from inside the Claude session: ask Claude to run `which me`. **MCP server shows "failed" in `/plugin`** -Usually means api_key or server is missing from userConfig. Fix the configuration, then pick "Reconnect" from the plugin menu (or restart the session). +Usually means there are no credentials to resolve: you're not logged in (`me login`) and no api_key is set, or `me` isn't on PATH. Fix it, then pick "Reconnect" from the plugin menu (or restart the session). diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index d17a086..b43da41 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -23,7 +23,7 @@ const BASE_EVENT = { const CONFIG: HookConfig = { server: "https://api.example.com", - apiKey: "me.lookupid12345678.secret", + token: "me.lookupid12345678.secret", space: "eng123", treePrefix: "claude_code.sessions", }; @@ -241,7 +241,7 @@ describe("captureHookEvent", () => { // ============================================================================= describe("resolveHookConfigFromEnv", () => { - test("returns null when api_key is missing", () => { + test("returns null when no api_key and no session fallback", () => { const cfg = resolveHookConfigFromEnv({}); expect(cfg).toBeNull(); }); @@ -261,7 +261,7 @@ describe("resolveHookConfigFromEnv", () => { CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "my.prefix", }); expect(cfg).toEqual({ - apiKey: "me.lookupid12345678.secret", + token: "me.lookupid12345678.secret", space: "eng123def456", server: "https://api.example.com", treePrefix: "my.prefix", @@ -274,13 +274,56 @@ describe("resolveHookConfigFromEnv", () => { CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", }); expect(cfg).toEqual({ - apiKey: "me.lookupid12345678.secret", + token: "me.lookupid12345678.secret", space: "eng123def456", server: "https://api.memory.build", treePrefix: "claude_code.sessions", }); }); + test("falls back to the login session when api_key is blank", () => { + const cfg = resolveHookConfigFromEnv( + { CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456" }, + { sessionToken: "sess-token", server: "https://api.example.com" }, + ); + expect(cfg).toEqual({ + token: "sess-token", + space: "eng123def456", + server: "https://api.example.com", + treePrefix: "claude_code.sessions", + }); + }); + + test("treats an unsubstituted ${...} placeholder as blank (uses session)", () => { + const cfg = resolveHookConfigFromEnv( + { + CLAUDE_PLUGIN_OPTION_API_KEY: "${user_config.api_key}", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", + }, + { sessionToken: "sess-token" }, + ); + expect(cfg?.token).toBe("sess-token"); + }); + + test("uses the active space fallback when plugin space is unset", () => { + const cfg = resolveHookConfigFromEnv( + {}, + { sessionToken: "sess-token", activeSpace: "act123def456" }, + ); + expect(cfg?.space).toBe("act123def456"); + }); + + test("plugin api_key takes precedence over the session", () => { + const cfg = resolveHookConfigFromEnv( + { + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", + }, + { sessionToken: "sess-token" }, + ); + expect(cfg?.token).toBe("me.lookupid12345678.secret"); + }); + test("treats empty string as missing (falls back to default)", () => { const cfg = resolveHookConfigFromEnv({ CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index 4ff6107..d469a1b 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -16,8 +16,12 @@ import { createMemoryClient, type MemoryClient } from "../client.ts"; export interface HookConfig { /** Memory Engine server URL. */ server: string; - /** Agent api key (from the plugin's sensitive userConfig). */ - apiKey: string; + /** + * Bearer for the memory endpoint: the plugin's api key (sensitive userConfig) + * when set, else the user's `me login` session token. Both authenticate the + * memory endpoint. + */ + token: string; /** Active space slug (X-Me-Space). */ space: string; /** Tree path prefix for captured memories (ltree). */ @@ -154,28 +158,59 @@ export function buildMeta( export const DEFAULT_SERVER = "https://api.memory.build"; export const DEFAULT_TREE_PREFIX = "claude_code.sessions"; +/** Credentials the hook falls back to when the plugin's api_key is unset. */ +export interface HookFallbackCreds { + apiKey?: string; + sessionToken?: string; + activeSpace?: string; + server?: string; +} + +/** + * Treat unset / empty / unsubstituted-placeholder values as missing. Claude Code + * may substitute an empty string (or leave the literal `${user_config.x}`) for an + * optional userConfig field the user left blank. + */ +function blank(v: string | undefined): boolean { + return !v || /^\$\{.*\}$/.test(v); +} + /** - * Resolve the hook config from `CLAUDE_PLUGIN_OPTION_*` env vars exported - * by Claude Code for the plugin. Returns null if required values are - * missing. + * Resolve the hook config. The bearer is the plugin's `api_key` (sensitive + * userConfig, delivered via `CLAUDE_PLUGIN_OPTION_API_KEY`) when set; otherwise + * it falls back to the user's `me login` session (passed in via `creds`, so this + * function stays pure/testable). The space comes from the plugin config, else the + * caller's active space. Returns null when no bearer or no space is available. * - * Claude Code delivers `sensitive: true` userConfig values (like api_key) - * through the same env var mechanism as non-sensitive ones. + * Claude Code delivers `sensitive: true` userConfig values (like api_key) through + * the same env var mechanism as non-sensitive ones. */ export function resolveHookConfigFromEnv( env: NodeJS.ProcessEnv = process.env, + creds: HookFallbackCreds = {}, ): HookConfig | null { - const apiKey = env.CLAUDE_PLUGIN_OPTION_API_KEY; - if (!apiKey) return null; - - // Api keys are global, so the space must be configured explicitly. - const space = env.CLAUDE_PLUGIN_OPTION_SPACE; + const pluginKey = blank(env.CLAUDE_PLUGIN_OPTION_API_KEY) + ? undefined + : env.CLAUDE_PLUGIN_OPTION_API_KEY; + // Bearer precedence mirrors `me mcp`: plugin key > ME_API_KEY > login session. + const token = pluginKey ?? creds.apiKey ?? creds.sessionToken; + if (!token) return null; + + // Space: plugin config, else the active space. Required either way (api keys + // are global, and a session still needs a target space). + const space = blank(env.CLAUDE_PLUGIN_OPTION_SPACE) + ? creds.activeSpace + : env.CLAUDE_PLUGIN_OPTION_SPACE; if (!space) return null; + const server = blank(env.CLAUDE_PLUGIN_OPTION_SERVER) + ? (creds.server ?? DEFAULT_SERVER) + : (env.CLAUDE_PLUGIN_OPTION_SERVER as string); + return { - apiKey, + token, space, - server: env.CLAUDE_PLUGIN_OPTION_SERVER || DEFAULT_SERVER, + server, treePrefix: env.CLAUDE_PLUGIN_OPTION_TREE_PREFIX || DEFAULT_TREE_PREFIX, }; } @@ -222,7 +257,7 @@ export async function captureHookEvent( opts.client ?? createMemoryClient({ url: config.server, - token: config.apiKey, + token: config.token, space: config.space, }); diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 077aff3..27f8447 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -9,10 +9,12 @@ * claude plugin marketplace add timescale/memory-engine * claude plugin install memory-engine@memory-engine [--scope user|project|local] * # then, in a Claude Code session: - * /plugin # select memory-engine, Configure, fill api_key/server/tree_prefix + * /plugin # select memory-engine, Configure, fill space (+ optional api_key) * * Claude Code delivers the configured values to our hook (`me claude - * hook --event `) via CLAUDE_PLUGIN_OPTION_* env vars. + * hook --event `) via CLAUDE_PLUGIN_OPTION_* env vars. api_key is + * optional: left blank, the hook (and the plugin's MCP server) use your + * `me login` session. * * 2. MCP-only via `me claude install`. Registers `me` as an MCP server * with Claude Code (no hooks, no slash commands — just the tools). @@ -25,6 +27,7 @@ import { type HookEventName, resolveHookConfigFromEnv, } from "../claude/capture.ts"; +import { resolveCredentials } from "../credentials.ts"; import { claudeImporter } from "../importers/claude.ts"; import { type AgentInstallOptions, @@ -89,9 +92,9 @@ function createClaudeInstallCommand(): Command { * me claude hook — invoked by the Claude Code plugin to capture events as * memories. * - * Reads the event JSON from stdin, pulls credentials + config from the - * CLAUDE_PLUGIN_OPTION_* env vars that Claude Code exports for the plugin, - * and creates a memory. + * Reads the event JSON from stdin, resolves config from the CLAUDE_PLUGIN_OPTION_* + * env vars Claude Code exports for the plugin (falling back to the `me login` + * session when no api_key is configured), and creates a memory. * * Best-effort: logs failures to stderr but always exits 0 so that a hook * failure never blocks a Claude Code session. @@ -112,12 +115,16 @@ function createClaudeHookCommand(): Command { process.exit(0); } - // Resolve config from env - const config = resolveHookConfigFromEnv(); + // Resolve config: the plugin's api_key if configured, else fall back to + // the user's `me login` session (resolved from the keychain/config). + const config = resolveHookConfigFromEnv( + process.env, + resolveCredentials(), + ); if (!config) { console.error( - "[memory-engine] CLAUDE_PLUGIN_OPTION_API_KEY not set. " + - "Configure the plugin via `/plugin` in Claude Code.", + "[memory-engine] no credentials. Run `me login`, or set the plugin's " + + "api_key + space via `/plugin` in Claude Code.", ); process.exit(0); } diff --git a/packages/cli/commands/mcp.ts b/packages/cli/commands/mcp.ts index 4224e61..a14e0b9 100644 --- a/packages/cli/commands/mcp.ts +++ b/packages/cli/commands/mcp.ts @@ -35,14 +35,25 @@ export function isLegacyApiKey(token: string): boolean { ); } +/** + * Treat unset / empty / unsubstituted-placeholder flag values as missing. The + * Claude Code plugin's .mcp.json substitutes `${user_config.api_key}` statically; + * when api_key is left blank that arrives as `""` (or the literal placeholder), + * which must fall through to the session, not be used as a token. + */ +function blankFlag(v: unknown): string | undefined { + if (typeof v !== "string" || v === "" || /^\$\{.*\}$/.test(v)) + return undefined; + return v; +} + function createMcpRunAction() { return async (_opts: Record, cmd: Command) => { const opts = cmd.optsWithGlobals(); const creds = resolveCredentials(opts.server as string | undefined); // Token: --api-key > ME_API_KEY (creds.apiKey) > stored session token. - const token = - (opts.apiKey as string | undefined) ?? creds.apiKey ?? creds.sessionToken; + const token = blankFlag(opts.apiKey) ?? creds.apiKey ?? creds.sessionToken; if (!token) { console.error( "Error: no credentials. Run 'me login', or pass --api-key / set ME_API_KEY.", @@ -60,7 +71,7 @@ function createMcpRunAction() { } // Space: --space > ME_SPACE / stored active space. - const space = (opts.space as string | undefined) ?? creds.activeSpace; + const space = blankFlag(opts.space) ?? creds.activeSpace; if (!space) { console.error( "Error: no active space. Run 'me space use ', or pass --space / set ME_SPACE.", From d2bdd9931bb225e3a037b44772a24c2b451a2f82 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 10:20:57 +0200 Subject: [PATCH 106/156] feat(claude-plugin): make space optional too, fall back to active space MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For parity with `me mcp` / install (which resolve the space at runtime), the plugin's `space` is no longer required: blank falls back to your active space (me space use / ME_SPACE). The hook and `me mcp` already supported the fallback; this flips required→false in plugin.json and documents it. Pin space for unattended or project-scope installs — a blank space with no active space set means captures are silently skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/cli/me-claude.md | 4 ++-- packages/claude-plugin/.claude-plugin/plugin.json | 6 +++--- packages/claude-plugin/README.md | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index da9963d..a535971 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -57,10 +57,10 @@ This command is not run directly -- the Claude Code plugin calls it. The plugin claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] # then, in a Claude Code session: -/plugin # select memory-engine, Configure, set space (api_key optional) +/plugin # select memory-engine, Configure (all values optional if logged in) ``` -`api_key` is optional: leave it blank and the plugin's hooks and MCP server use your `me login` session; set it to attribute captures to a dedicated agent instead. +Both `api_key` and `space` are optional: blank `api_key` uses your `me login` session (set it to attribute captures to a dedicated agent), and blank `space` uses your active space (`me space use`; pin it for project/shared installs). If you only want the MCP tools (no hooks, no slash commands), use [me claude install](#me-claude-install) instead. diff --git a/packages/claude-plugin/.claude-plugin/plugin.json b/packages/claude-plugin/.claude-plugin/plugin.json index 52b1abe..df9956b 100644 --- a/packages/claude-plugin/.claude-plugin/plugin.json +++ b/packages/claude-plugin/.claude-plugin/plugin.json @@ -26,9 +26,9 @@ }, "space": { "type": "string", - "title": "Space", - "description": "Space slug to store memories in (the X-Me-Space). Required — the active space isn't read from your CLI config here.", - "required": true + "title": "Space (optional)", + "description": "Space slug to capture into (the X-Me-Space). Leave blank to use your active space (`me space use` / ME_SPACE). Pin it for unattended or project-scope installs so captures always land in the same space; if blank and no active space is set, captures are skipped.", + "required": false }, "tree_prefix": { "type": "string", diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index f481aa2..8296075 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -64,20 +64,20 @@ claude plugin install memory-engine@memory-engine --scope local # this repo, ## Configure -The only required value is `space`. Claude Code does not prompt at install time — you configure from inside a session. +Every value is optional if you're logged in with an active space — the plugin falls back to your `me login` session and `me space use` space. Claude Code does not prompt at install time; configure from inside a session. ```text claude # start a session /plugin # open the plugin manager # → Installed → memory-engine → Configure -# → space (the space slug — REQUIRED) +# → space (OPTIONAL — blank = your active space; pin for project/shared installs) # → api_key (OPTIONAL, sensitive — blank = use your `me login` session) # → server (default https://api.memory.build) # → tree_prefix (default claude_code.sessions) # → values take effect immediately; no restart required ``` -Leave `api_key` blank to use your `me login` session (captures attributed to you); set it to use a dedicated agent key (see above). Sensitive values (the api_key) go to your system keychain; non-sensitive values go to the `settings.json` for the scope you installed in. +Leave `api_key` blank to use your `me login` session (captures attributed to you); set it to use a dedicated agent key (see above). Leave `space` blank to capture into your active space; pin it for unattended or project-scope installs (a blank space with no active space set means captures are silently skipped). Sensitive values (the api_key) go to your system keychain; non-sensitive values go to the `settings.json` for the scope you installed in. ## Verify @@ -138,8 +138,8 @@ Claude Code handles the cleanup. Your captured memories and API keys are preserv **`[memory-engine] no credentials` in stderr** The hook ran but found neither a `me login` session nor a configured api_key. Run `me login` (and `me space use `), or open `/plugin → memory-engine → Configure` and set the api_key + space. -**`Plugin option "X" isn't set` in Claude Code's error panel** -A required userConfig value is missing — `space` is required (api_key is optional). Configure it via `/plugin → memory-engine → Configure`. +**Hook fires but no memories appear, no error** +With everything optional, a hook silently skips when it can't resolve a space — no `space` configured *and* no active space set (`me space use`). Either pin `space` in `/plugin → Configure` or run `me space use `. **Hook fires but no memories appear** - Confirm the api_key is valid: From a86f8c4207a4a700a599ada9f3aa33f1efa65434 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 10:26:30 +0200 Subject: [PATCH 107/156] fix(cli): default capture/import paths under `share` so sessions can write them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin capture prefix (claude_code.sessions) and the import tree-root (projects) were top-level ltree paths. A session-authenticated user holds only owner@home (~) and access to `share` — neither covers a bare top-level path — so with the plugin now defaulting to the login session, captures/imports there would be forbidden. Move both defaults under `share`: - capture/plugin tree_prefix: claude_code.sessions → share.claude_code.session - import --tree-root default: projects → share.projects Update tests and the plugin/import docs (grant example, search filter, examples). Override either with --tree-root / tree_prefix for a different writable path. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/cli/agent-session-imports.md | 4 ++-- packages/claude-plugin/.claude-plugin/plugin.json | 4 ++-- packages/claude-plugin/README.md | 8 ++++---- packages/cli/claude/capture.test.ts | 10 +++++----- packages/cli/claude/capture.ts | 4 +++- packages/cli/commands/import.test.ts | 2 +- packages/cli/commands/import.ts | 5 ++++- 7 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/cli/agent-session-imports.md b/docs/cli/agent-session-imports.md index 3da6250..15c2859 100644 --- a/docs/cli/agent-session-imports.md +++ b/docs/cli/agent-session-imports.md @@ -18,7 +18,7 @@ All three subcommands accept the same flags (with one extra flag on `me claude i | `--project ` | Only import sessions whose cwd equals or is below this path. | | `--since ` | Only import sessions started at or after this ISO 8601 timestamp. | | `--until ` | Only import sessions started at or before this ISO 8601 timestamp. | -| `--tree-root ` | Tree root under which `.` nodes are placed. Default: `projects`. Must match `[a-z0-9_]+(\.[a-z0-9_]+)*`. | +| `--tree-root ` | Tree root under which `.` nodes are placed. Default: `share.projects`. Must match `[a-z0-9_]+(\.[a-z0-9_]+)*`. | | `--sessions-node-name ` | Per-project node name for imported agent sessions. Default: `agent_sessions`. Must match `[a-z0-9_]+`. | | `--full-transcript` | Also store reasoning, tool calls, and tool results as their own message memories (default: user + assistant text only). | | `--include-temp-cwd` | Include sessions whose cwd is a system temp directory (`/tmp`, `/private/var/folders/...`). Off by default. | @@ -40,7 +40,7 @@ Each imported message is stored under: .. ``` -For example, a Claude message from a session run in `/Users/me/dev/memory-engine` ends up under `projects.memory_engine.agent_sessions` by default. Every message from every session in a project shares that same tree node; individual sessions are distinguished by `meta.source_session_id`. +For example, a Claude message from a session run in `/Users/me/dev/memory-engine` ends up under `share.projects.memory_engine.agent_sessions` by default. Every message from every session in a project shares that same tree node; individual sessions are distinguished by `meta.source_session_id`. Project slugs come from the git repo root directory name when the cwd is inside a repo, or from `basename(cwd)` otherwise. Slug collisions (two different cwds that normalize to the same label) are resolved automatically by appending a 4-char hash suffix -- the first cwd seen gets the plain slug, subsequent ones get `slug_`. The full cwd is always preserved in `meta.source_cwd`. diff --git a/packages/claude-plugin/.claude-plugin/plugin.json b/packages/claude-plugin/.claude-plugin/plugin.json index df9956b..5dff6c4 100644 --- a/packages/claude-plugin/.claude-plugin/plugin.json +++ b/packages/claude-plugin/.claude-plugin/plugin.json @@ -33,8 +33,8 @@ "tree_prefix": { "type": "string", "title": "Tree prefix", - "description": "Ltree path prefix where captured prompts/responses are stored (e.g. `claude_code.sessions`).", - "default": "claude_code.sessions", + "description": "Ltree path prefix where captured prompts/responses are stored. Defaults under `share` so your login session can write there; use a `~`-prefixed path to keep captures in your private home instead.", + "default": "share.claude_code.session", "required": true } } diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index 8296075..7503683 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -39,7 +39,7 @@ me agent create claude-code-agent # 2. Add it to the space and grant just the access it needs — e.g. read+write on # the capture subtree (grants cover all descendant paths via ltree) me agent add claude-code-agent -me access grant claude-code-agent claude_code.sessions w +me access grant claude-code-agent share.claude_code.session w # 3. Mint an API key for that agent me apikey create claude-code-agent plugin-key @@ -73,7 +73,7 @@ claude # start a session # → space (OPTIONAL — blank = your active space; pin for project/shared installs) # → api_key (OPTIONAL, sensitive — blank = use your `me login` session) # → server (default https://api.memory.build) -# → tree_prefix (default claude_code.sessions) +# → tree_prefix (default share.claude_code.session) # → values take effect immediately; no restart required ``` @@ -84,12 +84,12 @@ Leave `api_key` blank to use your `me login` session (captures attributed to you After configuring, send a prompt in Claude Code, then check that capture happened: ```bash -me memory search --tree "claude_code.*" --limit 5 +me memory search --tree "share.claude_code.*" --limit 5 ``` You should see your recent prompts (and, after the agent finishes, its response). What gets stored: -- **Tree**: whatever you set in `tree_prefix` (default: `claude_code.sessions`) +- **Tree**: whatever you set in `tree_prefix` (default: `share.claude_code.session`) - **Metadata**: - `type`: `user_prompt` or `agent_response` - `session_id`: Claude Code's session UUID diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index b43da41..5c2581b 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -25,7 +25,7 @@ const CONFIG: HookConfig = { server: "https://api.example.com", token: "me.lookupid12345678.secret", space: "eng123", - treePrefix: "claude_code.sessions", + treePrefix: "share.claude_code.session", }; // ============================================================================= @@ -196,7 +196,7 @@ describe("captureHookEvent", () => { expect(calls).toHaveLength(1); const [call] = calls as [Record]; expect(call.content).toBe("hello"); - expect(call.tree).toBe("claude_code.sessions"); + expect(call.tree).toBe("share.claude_code.session"); expect(call.temporal).toEqual({ start: "2026-04-23T10:00:00.000Z" }); const meta = call.meta as Record; expect(meta.type).toBe("user_prompt"); @@ -277,7 +277,7 @@ describe("resolveHookConfigFromEnv", () => { token: "me.lookupid12345678.secret", space: "eng123def456", server: "https://api.memory.build", - treePrefix: "claude_code.sessions", + treePrefix: "share.claude_code.session", }); }); @@ -290,7 +290,7 @@ describe("resolveHookConfigFromEnv", () => { token: "sess-token", space: "eng123def456", server: "https://api.example.com", - treePrefix: "claude_code.sessions", + treePrefix: "share.claude_code.session", }); }); @@ -332,6 +332,6 @@ describe("resolveHookConfigFromEnv", () => { CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "", }); expect(cfg?.server).toBe("https://api.memory.build"); - expect(cfg?.treePrefix).toBe("claude_code.sessions"); + expect(cfg?.treePrefix).toBe("share.claude_code.session"); }); }); diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index d469a1b..389505b 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -156,7 +156,9 @@ export function buildMeta( // ============================================================================= export const DEFAULT_SERVER = "https://api.memory.build"; -export const DEFAULT_TREE_PREFIX = "claude_code.sessions"; +// Under `share` so a session-authenticated user (who holds owner@share, not +// access to arbitrary top-level paths) can actually write captures here. +export const DEFAULT_TREE_PREFIX = "share.claude_code.session"; /** Credentials the hook falls back to when the plugin's api_key is unset. */ export interface HookFallbackCreds { diff --git a/packages/cli/commands/import.test.ts b/packages/cli/commands/import.test.ts index 75dbc12..dee2055 100644 --- a/packages/cli/commands/import.test.ts +++ b/packages/cli/commands/import.test.ts @@ -5,7 +5,7 @@ describe("buildOptions", () => { test("defaults imported session node name to agent_sessions", () => { const config = buildOptions({}); - expect(config.write.treeRoot).toBe("projects"); + expect(config.write.treeRoot).toBe("share.projects"); expect(config.write.sessionsNodeName).toBe("agent_sessions"); }); diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index 54ccc3e..ff39dcc 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -41,7 +41,10 @@ import { requireSpace, } from "../util.ts"; -const DEFAULT_TREE_ROOT = "projects"; +// Under `share` so a session-authenticated user can write the import (they hold +// owner@share, not access to arbitrary top-level paths). Override with +// --tree-root for a different destination you have write access to. +const DEFAULT_TREE_ROOT = "share.projects"; const DEFAULT_SESSIONS_NODE_NAME = "agent_sessions"; const VALID_TREE_ROOT_RE = /^[a-z0-9_]+(\.[a-z0-9_]+)*$/; const VALID_TREE_LABEL_RE = /^[a-z0-9_]+$/; From ced6fb76accd511f37518ea654754e9f3eee1aa2 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 10:35:39 +0200 Subject: [PATCH 108/156] fix(cli): allow ~ (and lenient forms) in import --tree-root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --tree-root validation regex ([a-z0-9_]+(\.…)) rejected a leading `~`, so you couldn't import into your home tree. Relax it to the protocol's lenient input form (ltree labels separated by `.` or `/`, optional leading `~`); the server still normalizes + authoritatively validates. e.g. `--tree-root ~.projects` now works. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/cli/agent-session-imports.md | 2 +- packages/cli/commands/import.test.ts | 15 +++++++++++++++ packages/cli/commands/import.ts | 8 ++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/cli/agent-session-imports.md b/docs/cli/agent-session-imports.md index 15c2859..793af8a 100644 --- a/docs/cli/agent-session-imports.md +++ b/docs/cli/agent-session-imports.md @@ -18,7 +18,7 @@ All three subcommands accept the same flags (with one extra flag on `me claude i | `--project ` | Only import sessions whose cwd equals or is below this path. | | `--since ` | Only import sessions started at or after this ISO 8601 timestamp. | | `--until ` | Only import sessions started at or before this ISO 8601 timestamp. | -| `--tree-root ` | Tree root under which `.` nodes are placed. Default: `share.projects`. Must match `[a-z0-9_]+(\.[a-z0-9_]+)*`. | +| `--tree-root ` | Tree root under which `.` nodes are placed. Default: `share.projects`. Accepts ltree labels (`[A-Za-z0-9_-]`) separated by `.` or `/`, with an optional leading `~` for your home (e.g. `~.projects`). | | `--sessions-node-name ` | Per-project node name for imported agent sessions. Default: `agent_sessions`. Must match `[a-z0-9_]+`. | | `--full-transcript` | Also store reasoning, tool calls, and tool results as their own message memories (default: user + assistant text only). | | `--include-temp-cwd` | Include sessions whose cwd is a system temp directory (`/tmp`, `/private/var/folders/...`). Off by default. | diff --git a/packages/cli/commands/import.test.ts b/packages/cli/commands/import.test.ts index dee2055..b0c5862 100644 --- a/packages/cli/commands/import.test.ts +++ b/packages/cli/commands/import.test.ts @@ -20,4 +20,19 @@ describe("buildOptions", () => { "Invalid --sessions-node-name: 'agent-sessions'. Must match [a-z0-9_]+", ); }); + + test("accepts a ~ (home) tree root and other lenient forms", () => { + expect(buildOptions({ treeRoot: "~" }).write.treeRoot).toBe("~"); + expect(buildOptions({ treeRoot: "~.work" }).write.treeRoot).toBe("~.work"); + expect(buildOptions({ treeRoot: "~/work" }).write.treeRoot).toBe("~/work"); + expect(buildOptions({ treeRoot: "share.projects" }).write.treeRoot).toBe( + "share.projects", + ); + }); + + test("rejects a tree root with illegal characters", () => { + expect(() => buildOptions({ treeRoot: "bad space" })).toThrow( + "Invalid --tree-root", + ); + }); }); diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index ff39dcc..8efbb2b 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -46,7 +46,11 @@ import { // --tree-root for a different destination you have write access to. const DEFAULT_TREE_ROOT = "share.projects"; const DEFAULT_SESSIONS_NODE_NAME = "agent_sessions"; -const VALID_TREE_ROOT_RE = /^[a-z0-9_]+(\.[a-z0-9_]+)*$/; +// Lenient user-facing tree-path input (matches the protocol's treePathSchema): +// labels [A-Za-z0-9_-], `.` or `/` separators, optional leading `~` (home). The +// server normalizes + authoritatively validates; this is a fast pre-check so a +// `--tree-root ~/work` or `~` lands in the caller's home instead of being rejected. +const VALID_TREE_ROOT_RE = /^[A-Za-z0-9_~./-]+$/; const VALID_TREE_LABEL_RE = /^[a-z0-9_]+$/; /** Build a Commander option set shared by every subcommand. */ @@ -118,7 +122,7 @@ export function buildOptions(opts: Record): { : DEFAULT_SESSIONS_NODE_NAME; if (!VALID_TREE_ROOT_RE.test(treeRoot)) { throw new Error( - `Invalid --tree-root: '${treeRoot}'. Must match [a-z0-9_]+(\\.[a-z0-9_]+)*`, + `Invalid --tree-root: '${treeRoot}'. Use ltree labels ([A-Za-z0-9_-]) separated by '.' or '/', with an optional leading '~' for your home.`, ); } if (!VALID_TREE_LABEL_RE.test(sessionsNodeName)) { From 05b3fff413caf7ffdd0e3db32e120d11ec85d005 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 10:46:19 +0200 Subject: [PATCH 109/156] feat(claude-plugin): nest captures by project, aligned with the import tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live-capture hook wrote a flat path (share.claude_code.session) while the import tool nests by project (share.projects..agent_sessions). Align them: the hook now nests too, writing ..agent_sessions using the project it already derives (git remote / cwd). With the default root share.projects, live captures and imported sessions land in the SAME node per project, distinguished by meta.source. - capture.ts: HookConfig.treePrefix → treeRoot; DEFAULT_TREE_PREFIX → DEFAULT_TREE_ROOT="share.projects"; fixed SESSIONS_NODE="agent_sessions" leaf (matches import's --sessions-node-name default); tree built per-project. - plugin.json: userConfig tree_prefix → tree_root (default share.projects), CLAUDE_PLUGIN_OPTION_TREE_ROOT. - Update tests + plugin/getting-started/mcp docs. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/cli/me-mcp.md | 2 +- docs/getting-started.md | 2 +- docs/mcp-integration.md | 2 +- .../claude-plugin/.claude-plugin/plugin.json | 8 ++--- packages/claude-plugin/README.md | 8 ++--- packages/cli/claude/capture.test.ts | 26 ++++++++-------- packages/cli/claude/capture.ts | 30 +++++++++++++------ 7 files changed, 46 insertions(+), 32 deletions(-) diff --git a/docs/cli/me-mcp.md b/docs/cli/me-mcp.md index 2b84416..29383de 100644 --- a/docs/cli/me-mcp.md +++ b/docs/cli/me-mcp.md @@ -46,4 +46,4 @@ claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] ``` -Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `api_key`, `server`, and `tree_prefix`. +Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `space` (and optionally `api_key`, `server`, `tree_root`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 8c6c722..e167156 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -79,7 +79,7 @@ claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine ``` -Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `api_key`, `server`, and `tree_prefix`. +Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `space` (and optionally `api_key`, `server`, `tree_root`). After installation, your AI agent has access to memory tools -- create, search, get, update, delete, and more. diff --git a/docs/mcp-integration.md b/docs/mcp-integration.md index bd37416..0318ce0 100644 --- a/docs/mcp-integration.md +++ b/docs/mcp-integration.md @@ -50,7 +50,7 @@ claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] ``` -Claude Code uses the Memory Engine plugin. After installing it, start a Claude Code session, run `/plugin`, select `memory-engine`, and configure `api_key`, `server`, and `tree_prefix`. The plugin provides the MCP server and captures Claude Code session events as memories. +Claude Code uses the Memory Engine plugin. After installing it, start a Claude Code session, run `/plugin`, select `memory-engine`, and configure `space` (and optionally `api_key`, `server`, `tree_root`). The plugin provides the MCP server and captures Claude Code session events as memories. ### Gemini CLI diff --git a/packages/claude-plugin/.claude-plugin/plugin.json b/packages/claude-plugin/.claude-plugin/plugin.json index 5dff6c4..13e3837 100644 --- a/packages/claude-plugin/.claude-plugin/plugin.json +++ b/packages/claude-plugin/.claude-plugin/plugin.json @@ -30,11 +30,11 @@ "description": "Space slug to capture into (the X-Me-Space). Leave blank to use your active space (`me space use` / ME_SPACE). Pin it for unattended or project-scope installs so captures always land in the same space; if blank and no active space is set, captures are skipped.", "required": false }, - "tree_prefix": { + "tree_root": { "type": "string", - "title": "Tree prefix", - "description": "Ltree path prefix where captured prompts/responses are stored. Defaults under `share` so your login session can write there; use a `~`-prefixed path to keep captures in your private home instead.", - "default": "share.claude_code.session", + "title": "Tree root", + "description": "Ltree root under which captures are nested as `..agent_sessions` (the same layout `me ... import` uses, so live and imported sessions share a node per project). Defaults under `share` so your login session can write there; use a `~`-prefixed root to keep captures in your private home instead.", + "default": "share.projects", "required": true } } diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index 7503683..609b5df 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -39,7 +39,7 @@ me agent create claude-code-agent # 2. Add it to the space and grant just the access it needs — e.g. read+write on # the capture subtree (grants cover all descendant paths via ltree) me agent add claude-code-agent -me access grant claude-code-agent share.claude_code.session w +me access grant claude-code-agent share.projects w # 3. Mint an API key for that agent me apikey create claude-code-agent plugin-key @@ -73,7 +73,7 @@ claude # start a session # → space (OPTIONAL — blank = your active space; pin for project/shared installs) # → api_key (OPTIONAL, sensitive — blank = use your `me login` session) # → server (default https://api.memory.build) -# → tree_prefix (default share.claude_code.session) +# → tree_root (default share.projects; captures nest at ..agent_sessions) # → values take effect immediately; no restart required ``` @@ -84,12 +84,12 @@ Leave `api_key` blank to use your `me login` session (captures attributed to you After configuring, send a prompt in Claude Code, then check that capture happened: ```bash -me memory search --tree "share.claude_code.*" --limit 5 +me memory search --tree "share.projects.*" --limit 5 ``` You should see your recent prompts (and, after the agent finishes, its response). What gets stored: -- **Tree**: whatever you set in `tree_prefix` (default: `share.claude_code.session`) +- **Tree**: `..agent_sessions` (default root `share.projects`) — same layout as `me … import`, so live + imported sessions share a node per project - **Metadata**: - `type`: `user_prompt` or `agent_response` - `session_id`: Claude Code's session UUID diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index 5c2581b..0da5bcf 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -25,7 +25,7 @@ const CONFIG: HookConfig = { server: "https://api.example.com", token: "me.lookupid12345678.secret", space: "eng123", - treePrefix: "share.claude_code.session", + treeRoot: "share.projects", }; // ============================================================================= @@ -196,9 +196,10 @@ describe("captureHookEvent", () => { expect(calls).toHaveLength(1); const [call] = calls as [Record]; expect(call.content).toBe("hello"); - expect(call.tree).toBe("share.claude_code.session"); expect(call.temporal).toEqual({ start: "2026-04-23T10:00:00.000Z" }); const meta = call.meta as Record; + // Nested as ..agent_sessions (project derived from cwd). + expect(call.tree).toBe(`share.projects.${meta.project}.agent_sessions`); expect(meta.type).toBe("user_prompt"); expect(meta.session_id).toBe("sess-abc"); expect(meta.source).toBe("claude-code"); @@ -221,18 +222,19 @@ describe("captureHookEvent", () => { expect(meta.type).toBe("agent_response"); }); - test("uses custom treePrefix from config", async () => { + test("nests under a custom treeRoot from config", async () => { const { client, calls } = mockClient(); const event: HookEvent = { ...BASE_EVENT, prompt: "x" }; const cfg: HookConfig = { ...CONFIG, - treePrefix: "my.custom.prefix", + treeRoot: "~.work", }; await captureHookEvent(event, "user-prompt-submit", cfg, { client }); const [call] = calls as [Record]; - expect(call.tree).toBe("my.custom.prefix"); + const meta = call.meta as Record; + expect(call.tree).toBe(`~.work.${meta.project}.agent_sessions`); }); }); @@ -258,17 +260,17 @@ describe("resolveHookConfigFromEnv", () => { CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", CLAUDE_PLUGIN_OPTION_SERVER: "https://api.example.com", - CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "my.prefix", + CLAUDE_PLUGIN_OPTION_TREE_ROOT: "my.root", }); expect(cfg).toEqual({ token: "me.lookupid12345678.secret", space: "eng123def456", server: "https://api.example.com", - treePrefix: "my.prefix", + treeRoot: "my.root", }); }); - test("falls back to default server and tree_prefix", () => { + test("falls back to default server and tree_root", () => { const cfg = resolveHookConfigFromEnv({ CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", @@ -277,7 +279,7 @@ describe("resolveHookConfigFromEnv", () => { token: "me.lookupid12345678.secret", space: "eng123def456", server: "https://api.memory.build", - treePrefix: "share.claude_code.session", + treeRoot: "share.projects", }); }); @@ -290,7 +292,7 @@ describe("resolveHookConfigFromEnv", () => { token: "sess-token", space: "eng123def456", server: "https://api.example.com", - treePrefix: "share.claude_code.session", + treeRoot: "share.projects", }); }); @@ -329,9 +331,9 @@ describe("resolveHookConfigFromEnv", () => { CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", CLAUDE_PLUGIN_OPTION_SERVER: "", - CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "", + CLAUDE_PLUGIN_OPTION_TREE_ROOT: "", }); expect(cfg?.server).toBe("https://api.memory.build"); - expect(cfg?.treePrefix).toBe("share.claude_code.session"); + expect(cfg?.treeRoot).toBe("share.projects"); }); }); diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index 389505b..064fc29 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -24,8 +24,12 @@ export interface HookConfig { token: string; /** Active space slug (X-Me-Space). */ space: string; - /** Tree path prefix for captured memories (ltree). */ - treePrefix: string; + /** + * Tree root under which captures are nested as + * `..` — the same shape the import tool + * writes, so live captures and imported sessions share one node per project. + */ + treeRoot: string; } // ============================================================================= @@ -156,9 +160,15 @@ export function buildMeta( // ============================================================================= export const DEFAULT_SERVER = "https://api.memory.build"; -// Under `share` so a session-authenticated user (who holds owner@share, not -// access to arbitrary top-level paths) can actually write captures here. -export const DEFAULT_TREE_PREFIX = "share.claude_code.session"; +// Captures nest as `..`, identical to the +// import tool's default layout (see DEFAULT_TREE_ROOT / DEFAULT_SESSIONS_NODE_NAME +// in packages/cli/commands/import.ts), so live + imported sessions for a project +// land in the same node (distinguished by meta.source). Under `share` so a +// session-authenticated user (owner@share, not arbitrary top-level paths) can +// write here. +export const DEFAULT_TREE_ROOT = "share.projects"; +// Fixed per-project leaf, matching the import tool's --sessions-node-name default. +const SESSIONS_NODE = "agent_sessions"; /** Credentials the hook falls back to when the plugin's api_key is unset. */ export interface HookFallbackCreds { @@ -213,7 +223,7 @@ export function resolveHookConfigFromEnv( token, space, server, - treePrefix: env.CLAUDE_PLUGIN_OPTION_TREE_PREFIX || DEFAULT_TREE_PREFIX, + treeRoot: env.CLAUDE_PLUGIN_OPTION_TREE_ROOT || DEFAULT_TREE_ROOT, }; } @@ -237,8 +247,8 @@ export interface CaptureOptions { /** * Capture a hook event as a memory. * - * Returns immediately if there's no content to capture. Otherwise creates - * a memory in the engine under `config.treePrefix` with metadata. + * Returns immediately if there's no content to capture. Otherwise creates a + * memory under `..agent_sessions` with metadata. */ export async function captureHookEvent( event: HookEvent, @@ -263,9 +273,11 @@ export async function captureHookEvent( space: config.space, }); + // Nest by project under the configured root, with a fixed sessions leaf — + // `..` — matching the import tool's layout. const result = await client.memory.create({ content, - tree: config.treePrefix, + tree: `${config.treeRoot}.${project}.${SESSIONS_NODE}`, meta, temporal: { start: now.toISOString() }, }); From 5f6585dee0b6b1d6161e3f39d97cd79ea1cdb874 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 10:55:51 +0200 Subject: [PATCH 110/156] refactor(cli): share project-slug derivation between hook and import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capture hook and the import tool derived the project label with separate, divergent logic — the hook keyed off the git remote repo name, the import off the repo-root directory name — so the same repo could resolve to different slugs and the live/imported sessions would NOT co-locate (defeating the path alignment). Unify on the git `origin` remote repo name for both (your call), via shared code in importers/slug.ts: - add repoNameFromRemote() + deriveBaseSlug() (remote name > repo-root dir name > basename(cwd), normalized) and a single-shot resolveProjectSlug() for the hook. - SlugRegistry now derives its base slug the same way (import base label shifts from dir name to remote name; collision suffixing unchanged). - capture.ts drops its local deriveProject/sanitizeLtreeLabel and awaits the shared resolveProjectSlug. Tests moved/added accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/claude/capture.test.ts | 27 ++------------- packages/cli/claude/capture.ts | 51 ++++------------------------- packages/cli/importers/slug.test.ts | 33 ++++++++++++++++++- packages/cli/importers/slug.ts | 46 +++++++++++++++++++++++--- 4 files changed, 82 insertions(+), 75 deletions(-) diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index 0da5bcf..4270918 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -6,7 +6,6 @@ import type { MemoryClient } from "@memory.build/client"; import { buildMeta, captureHookEvent, - deriveProject, extractContent, type HookConfig, type HookEvent, @@ -94,30 +93,8 @@ describe("metaTypeForEvent", () => { }); }); -// ============================================================================= -// deriveProject -// ============================================================================= - -describe("deriveProject", () => { - test("falls back to cwd basename when git is unavailable", () => { - // /tmp/__nonexistent-dir-for-test__ has no git remote - const project = deriveProject("/tmp/myproject"); - // Can't assert exact value — may hit a git repo if /tmp is one. - // But the result should be a lowercase, sanitized single label. - expect(project).toMatch(/^[a-z0-9_]+$/); - }); - - test("handles empty cwd with 'unknown' fallback", () => { - const project = deriveProject(""); - expect(project).toMatch(/^[a-z0-9_]+$/); - }); - - test("sanitizes special characters in basename", () => { - const project = deriveProject("/tmp/my-proj.foo"); - // Result should only contain letters/digits/underscores - expect(project).toMatch(/^[a-z0-9_]+$/); - }); -}); +// (Project-slug derivation lives in importers/slug.ts — resolveProjectSlug — +// and is covered by slug.test.ts; the hook just consumes it.) // ============================================================================= // buildMeta diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index 064fc29..77e378c 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -1,13 +1,16 @@ /** * Claude Code hook event parsing and memory capture. * - * Pure functions for event parsing, project derivation, and metadata - * construction are testable in isolation. The `captureHookEvent` entry - * point handles memory creation via the memory client. + * Pure functions for event parsing and metadata construction are testable in + * isolation. The `captureHookEvent` entry point handles memory creation via the + * memory client. The project label is derived by the shared `resolveProjectSlug` + * (the same logic the import tool uses) so live + imported sessions for a repo + * share one project node. */ import { CLIENT_VERSION } from "../../../version"; import { createMemoryClient, type MemoryClient } from "../client.ts"; +import { resolveProjectSlug } from "../importers/slug.ts"; // ============================================================================= // Hook config (derived at runtime from CLAUDE_PLUGIN_OPTION_* env vars) @@ -99,46 +102,6 @@ export function metaTypeForEvent(eventName: HookEventName): string { } } -/** - * Normalize a raw string into a single ltree label. - * Letters, digits, and underscores only; lowercased. - */ -function sanitizeLtreeLabel(raw: string): string { - const cleaned = raw.replace(/[^a-zA-Z0-9]/g, "_").toLowerCase(); - return cleaned.length > 0 ? cleaned : "unknown"; -} - -/** - * Derive a project label from a cwd. - * - * Tries `git remote get-url origin` first; falls back to the basename of - * the cwd. The result is a single ltree label (sanitized). - */ -export function deriveProject(cwd: string): string { - try { - const proc = Bun.spawnSync(["git", "remote", "get-url", "origin"], { - cwd, - stdout: "pipe", - stderr: "ignore", - }); - if (proc.exitCode === 0) { - const url = new TextDecoder().decode(proc.stdout).trim(); - // Extract the last path segment, stripping .git - // Matches https://github.com/org/repo.git and git@github.com:org/repo.git - const match = url.match(/[/:]([^/:]+?)(?:\.git)?$/); - if (match?.[1]) { - return sanitizeLtreeLabel(match[1]); - } - } - } catch { - // Fall through to cwd basename - } - - const parts = cwd.split("/").filter(Boolean); - const basename = parts[parts.length - 1] ?? "unknown"; - return sanitizeLtreeLabel(basename); -} - /** Build the metadata object for a captured memory. */ export function buildMeta( event: HookEvent, @@ -261,7 +224,7 @@ export async function captureHookEvent( return { status: "skipped", reason: "empty content" }; } - const project = deriveProject(event.cwd); + const project = await resolveProjectSlug(event.cwd); const meta = buildMeta(event, eventName, project); const now = (opts.now ?? (() => new Date()))(); diff --git a/packages/cli/importers/slug.test.ts b/packages/cli/importers/slug.test.ts index 506cbbb..9c95886 100644 --- a/packages/cli/importers/slug.test.ts +++ b/packages/cli/importers/slug.test.ts @@ -6,7 +6,38 @@ * `undefined` and the fallback to `basename(cwd)` is exercised. */ import { describe, expect, test } from "bun:test"; -import { normalizeSlug, SlugRegistry } from "./slug.ts"; +import { + normalizeSlug, + repoNameFromRemote, + resolveProjectSlug, + SlugRegistry, +} from "./slug.ts"; + +describe("repoNameFromRemote", () => { + test("extracts the repo name from https and ssh remotes (sans .git)", () => { + expect(repoNameFromRemote("https://github.com/org/memory-engine.git")).toBe( + "memory-engine", + ); + expect(repoNameFromRemote("git@github.com:org/memory-engine.git")).toBe( + "memory-engine", + ); + expect(repoNameFromRemote("https://example.com/a/b/repo")).toBe("repo"); + }); +}); + +describe("resolveProjectSlug", () => { + test("returns 'unknown' for an empty/missing cwd", async () => { + expect(await resolveProjectSlug(undefined)).toBe("unknown"); + expect(await resolveProjectSlug("")).toBe("unknown"); + }); + + test("falls back to a normalized cwd basename when not in a git repo", async () => { + // /tmp/nonexistent-... isn't a git repo → no remote/root → basename. + expect( + await resolveProjectSlug("/tmp/nonexistent-path-xyz/memory-engine"), + ).toBe("memory_engine"); + }); +}); describe("normalizeSlug", () => { test("lowercases and replaces non-alphanumeric with underscore", () => { diff --git a/packages/cli/importers/slug.ts b/packages/cli/importers/slug.ts index daf25bd..fecfdb9 100644 --- a/packages/cli/importers/slug.ts +++ b/packages/cli/importers/slug.ts @@ -2,8 +2,10 @@ * Project slug derivation for agent conversation imports. * * A "slug" is an ltree-safe label derived from the session's cwd: - * - Prefer the git repo root directory name if the cwd is inside a repo. - * - Fall back to `basename(cwd)`. + * - Prefer the git `origin` remote's repo name (stable across clone locations, + * and shared with the Claude Code capture hook via `resolveProjectSlug`, so + * live + imported sessions for the same repo share one project label). + * - Else the git repo root directory name, else `basename(cwd)`. * - Normalize to `[a-z0-9_]+`. * * Slug collisions (different cwds that normalize to the same label) are @@ -54,6 +56,14 @@ export function normalizeSlug(raw: string): string { return collapsed; } +/** + * Extract the repo name (last path segment, sans `.git`) from a git remote URL. + * Handles `https://github.com/org/repo.git` and `git@github.com:org/repo.git`. + */ +export function repoNameFromRemote(url: string): string | undefined { + return url.trim().match(/[/:]([^/:]+?)(?:\.git)?$/)?.[1]; +} + /** * Detect the git top-level directory for `cwd`, if any. * @@ -126,6 +136,34 @@ function shortHash(input: string): string { return createHash("sha256").update(input).digest("hex").slice(0, 4); } +/** + * Derive the base (pre-collision) project slug for a cwd: the git `origin` + * remote's repo name if available, else the git repo root dir name, else + * `basename(cwd)` — normalized to an ltree label. Returns the git info too so + * callers can use it for collision disambiguation. + */ +async function deriveBaseSlug( + cwd: string, +): Promise<{ baseSlug: string; gitRoot?: string; gitRemote?: string }> { + const { gitRoot, gitRemote } = await getGitInfo(cwd); + const rawName = + (gitRemote ? repoNameFromRemote(gitRemote) : undefined) ?? + basename(gitRoot ?? cwd); + return { baseSlug: normalizeSlug(rawName), gitRoot, gitRemote }; +} + +/** + * Single-shot project slug for one cwd (no cross-run collision registry) — + * used by the Claude Code capture hook so live captures nest under the same + * project label the import tool uses. Returns the `unknown` slug for an + * empty/missing cwd. + */ +export async function resolveProjectSlug(cwd?: string): Promise { + if (!cwd || cwd.trim().length === 0) return UNKNOWN_SLUG; + const { baseSlug } = await deriveBaseSlug(cwd); + return baseSlug; +} + /** * Registry used to track slug assignments across a single import run so * colliding base slugs from different projects get distinct suffixes. @@ -149,9 +187,7 @@ export class SlugRegistry { return { slug: UNKNOWN_SLUG, baseSlug: UNKNOWN_SLUG, cwd: "" }; } - const { gitRoot, gitRemote } = await getGitInfo(cwd); - const source = gitRoot ?? cwd; - const baseSlug = normalizeSlug(basename(source)); + const { baseSlug, gitRoot, gitRemote } = await deriveBaseSlug(cwd); const bucket = this.assignments.get(baseSlug) ?? []; From 9219583198ede8748627bcb72ebf46639ebd1117 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 11:09:57 +0200 Subject: [PATCH 111/156] feat(claude-plugin): align capture metadata with the import schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-locating live captures and imports in one project node only helps if they're uniformly queryable, but their meta schemas barely overlapped (and `type` conflicted: user_prompt/agent_response vs agent_session). Align the hook's meta to the import's source_* convention — judged per field, adopting the import's where it's the better design and keeping the hook's where dictated by live capture: - type → "agent_session" (constant); role moves to source_message_role (user/ assistant) — resolves the conflict, one query finds all agent-session memory. - session_id→source_session_id, project→source_project_slug, cwd→source_cwd, source→source_tool; add content_mode="default" and source_git_repo (the slug lookup already fetches the remote, now returned from resolveProjectSlug). - Kept as-is (inherent to live capture, not bugs): server-generated id (no re-import to dedupe), temporal=now (≈ message time live), and capturing only the prompt + final response (hooks can't see intermediate messages). me_version stays (CLI version; a different axis from importer_version). metaTypeForEvent → messageRoleForEvent. Tests + plugin README updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/claude-plugin/README.md | 15 +++--- packages/cli/claude/capture.test.ts | 72 ++++++++++++++++++----------- packages/cli/claude/capture.ts | 42 ++++++++++------- packages/cli/importers/slug.test.ts | 14 +++--- packages/cli/importers/slug.ts | 16 ++++--- 5 files changed, 95 insertions(+), 64 deletions(-) diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index 609b5df..dd11fd6 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -90,12 +90,15 @@ me memory search --tree "share.projects.*" --limit 5 You should see your recent prompts (and, after the agent finishes, its response). What gets stored: - **Tree**: `..agent_sessions` (default root `share.projects`) — same layout as `me … import`, so live + imported sessions share a node per project -- **Metadata**: - - `type`: `user_prompt` or `agent_response` - - `session_id`: Claude Code's session UUID - - `project`: derived from `git remote get-url origin` in the session cwd (falls back to cwd basename) - - `cwd`: working directory when the hook fired - - `source`: `"claude-code"` +- **Metadata** (the same `source_*` schema `me … import` writes, so live + imported sessions are queryable together): + - `type`: `agent_session` + - `source_tool`: `"claude-code"` + - `source_session_id`: Claude Code's session UUID + - `source_message_role`: `user` or `assistant` + - `source_project_slug`: derived from the git `origin` remote (falls back to the cwd basename) + - `source_cwd`: working directory when the hook fired + - `source_git_repo`: the git remote URL (when the cwd is in a repo) + - `content_mode`: `default` - `me_version`: the `me` CLI version that created the memory - **Temporal**: ISO timestamp of hook invocation diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index 4270918..cbc55d4 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -9,7 +9,7 @@ import { extractContent, type HookConfig, type HookEvent, - metaTypeForEvent, + messageRoleForEvent, resolveHookConfigFromEnv, } from "./capture.ts"; @@ -80,16 +80,16 @@ describe("extractContent", () => { }); // ============================================================================= -// metaTypeForEvent +// messageRoleForEvent // ============================================================================= -describe("metaTypeForEvent", () => { - test("maps user-prompt-submit to user_prompt", () => { - expect(metaTypeForEvent("user-prompt-submit")).toBe("user_prompt"); +describe("messageRoleForEvent", () => { + test("maps user-prompt-submit to user", () => { + expect(messageRoleForEvent("user-prompt-submit")).toBe("user"); }); - test("maps stop to agent_response", () => { - expect(metaTypeForEvent("stop")).toBe("agent_response"); + test("maps stop to assistant", () => { + expect(messageRoleForEvent("stop")).toBe("assistant"); }); }); @@ -97,30 +97,42 @@ describe("metaTypeForEvent", () => { // and is covered by slug.test.ts; the hook just consumes it.) // ============================================================================= -// buildMeta +// buildMeta — aligned with the import tool's source_* schema // ============================================================================= describe("buildMeta", () => { - test("builds metadata with required fields", () => { + test("builds the source_* metadata for a user prompt", () => { const event: HookEvent = { ...BASE_EVENT, prompt: "hi" }; const meta = buildMeta(event, "user-prompt-submit", "myproject"); - expect(meta.type).toBe("user_prompt"); - expect(meta.session_id).toBe("sess-abc"); - expect(meta.cwd).toBe("/tmp/myproj"); - expect(meta.project).toBe("myproject"); - expect(meta.source).toBe("claude-code"); - expect(meta.me_version).toBeDefined(); + expect(meta.type).toBe("agent_session"); + expect(meta.source_tool).toBe("claude-code"); + expect(meta.source_session_id).toBe("sess-abc"); + expect(meta.source_message_role).toBe("user"); + expect(meta.source_project_slug).toBe("myproject"); + expect(meta.source_cwd).toBe("/tmp/myproj"); + expect(meta.content_mode).toBe("default"); expect(typeof meta.me_version).toBe("string"); + // No git remote passed → no source_git_repo. + expect(meta.source_git_repo).toBeUndefined(); }); - test("uses agent_response type for stop event", () => { - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: "done", - }; + test("uses the assistant role for a stop event", () => { + const event: HookEvent = { ...BASE_EVENT, last_assistant_message: "done" }; const meta = buildMeta(event, "stop", "proj"); - expect(meta.type).toBe("agent_response"); + expect(meta.type).toBe("agent_session"); + expect(meta.source_message_role).toBe("assistant"); + }); + + test("includes source_git_repo when a remote is provided", () => { + const event: HookEvent = { ...BASE_EVENT, prompt: "hi" }; + const meta = buildMeta( + event, + "user-prompt-submit", + "proj", + "git@github.com:org/repo.git", + ); + expect(meta.source_git_repo).toBe("git@github.com:org/repo.git"); }); }); @@ -176,13 +188,16 @@ describe("captureHookEvent", () => { expect(call.temporal).toEqual({ start: "2026-04-23T10:00:00.000Z" }); const meta = call.meta as Record; // Nested as ..agent_sessions (project derived from cwd). - expect(call.tree).toBe(`share.projects.${meta.project}.agent_sessions`); - expect(meta.type).toBe("user_prompt"); - expect(meta.session_id).toBe("sess-abc"); - expect(meta.source).toBe("claude-code"); + expect(call.tree).toBe( + `share.projects.${meta.source_project_slug}.agent_sessions`, + ); + expect(meta.type).toBe("agent_session"); + expect(meta.source_message_role).toBe("user"); + expect(meta.source_session_id).toBe("sess-abc"); + expect(meta.source_tool).toBe("claude-code"); }); - test("captures stop event with agent_response type", async () => { + test("captures stop event with the assistant role", async () => { const { client, calls } = mockClient(); const event: HookEvent = { ...BASE_EVENT, @@ -196,7 +211,8 @@ describe("captureHookEvent", () => { const [call] = calls as [Record]; expect(call.content).toBe("goodbye"); const meta = call.meta as Record; - expect(meta.type).toBe("agent_response"); + expect(meta.type).toBe("agent_session"); + expect(meta.source_message_role).toBe("assistant"); }); test("nests under a custom treeRoot from config", async () => { @@ -211,7 +227,7 @@ describe("captureHookEvent", () => { const [call] = calls as [Record]; const meta = call.meta as Record; - expect(call.tree).toBe(`~.work.${meta.project}.agent_sessions`); + expect(call.tree).toBe(`~.work.${meta.source_project_slug}.agent_sessions`); }); }); diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index 77e378c..0bf79c0 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -92,30 +92,38 @@ export function extractContent( return null; } -/** Map an event name to the `type` metadata value. */ -export function metaTypeForEvent(eventName: HookEventName): string { - switch (eventName) { - case "user-prompt-submit": - return "user_prompt"; - case "stop": - return "agent_response"; - } +/** Map an event name to the message role (`source_message_role`). */ +export function messageRoleForEvent( + eventName: HookEventName, +): "user" | "assistant" { + return eventName === "user-prompt-submit" ? "user" : "assistant"; } -/** Build the metadata object for a captured memory. */ +/** + * Build the metadata for a captured memory. Uses the same `source_*` schema as + * the import tool (see buildMeta in packages/cli/importers/index.ts) so live + * captures and imported sessions co-located in a project node are uniformly + * queryable. `type` is the constant `agent_session`; the prompt/response + * distinction lives in `source_message_role`. + */ export function buildMeta( event: HookEvent, eventName: HookEventName, project: string, + gitRepo?: string, ): Record { - return { - type: metaTypeForEvent(eventName), - session_id: event.session_id, - project, - cwd: event.cwd, - source: "claude-code", + const meta: Record = { + type: "agent_session", + source_tool: "claude-code", + source_session_id: event.session_id, + source_message_role: messageRoleForEvent(eventName), + source_project_slug: project, + source_cwd: event.cwd, + content_mode: "default", me_version: CLIENT_VERSION, }; + if (gitRepo) meta.source_git_repo = gitRepo; + return meta; } // ============================================================================= @@ -224,8 +232,8 @@ export async function captureHookEvent( return { status: "skipped", reason: "empty content" }; } - const project = await resolveProjectSlug(event.cwd); - const meta = buildMeta(event, eventName, project); + const { slug: project, gitRemote } = await resolveProjectSlug(event.cwd); + const meta = buildMeta(event, eventName, project, gitRemote); const now = (opts.now ?? (() => new Date()))(); const client = diff --git a/packages/cli/importers/slug.test.ts b/packages/cli/importers/slug.test.ts index 9c95886..6f5c0a7 100644 --- a/packages/cli/importers/slug.test.ts +++ b/packages/cli/importers/slug.test.ts @@ -26,16 +26,18 @@ describe("repoNameFromRemote", () => { }); describe("resolveProjectSlug", () => { - test("returns 'unknown' for an empty/missing cwd", async () => { - expect(await resolveProjectSlug(undefined)).toBe("unknown"); - expect(await resolveProjectSlug("")).toBe("unknown"); + test("returns 'unknown' (no remote) for an empty/missing cwd", async () => { + expect(await resolveProjectSlug(undefined)).toEqual({ slug: "unknown" }); + expect(await resolveProjectSlug("")).toEqual({ slug: "unknown" }); }); test("falls back to a normalized cwd basename when not in a git repo", async () => { // /tmp/nonexistent-... isn't a git repo → no remote/root → basename. - expect( - await resolveProjectSlug("/tmp/nonexistent-path-xyz/memory-engine"), - ).toBe("memory_engine"); + const { slug, gitRemote } = await resolveProjectSlug( + "/tmp/nonexistent-path-xyz/memory-engine", + ); + expect(slug).toBe("memory_engine"); + expect(gitRemote).toBeUndefined(); }); }); diff --git a/packages/cli/importers/slug.ts b/packages/cli/importers/slug.ts index fecfdb9..2df6f71 100644 --- a/packages/cli/importers/slug.ts +++ b/packages/cli/importers/slug.ts @@ -153,15 +153,17 @@ async function deriveBaseSlug( } /** - * Single-shot project slug for one cwd (no cross-run collision registry) — + * Single-shot project context for one cwd (no cross-run collision registry) — * used by the Claude Code capture hook so live captures nest under the same - * project label the import tool uses. Returns the `unknown` slug for an - * empty/missing cwd. + * project label the import tool uses, and can record the git remote. Returns the + * `unknown` slug (and no remote) for an empty/missing cwd. */ -export async function resolveProjectSlug(cwd?: string): Promise { - if (!cwd || cwd.trim().length === 0) return UNKNOWN_SLUG; - const { baseSlug } = await deriveBaseSlug(cwd); - return baseSlug; +export async function resolveProjectSlug( + cwd?: string, +): Promise<{ slug: string; gitRemote?: string }> { + if (!cwd || cwd.trim().length === 0) return { slug: UNKNOWN_SLUG }; + const { baseSlug, gitRemote } = await deriveBaseSlug(cwd); + return { slug: baseSlug, gitRemote }; } /** From e9a6eec5d1647171fb43def54462acaf67b669e4 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 12:20:31 +0200 Subject: [PATCH 112/156] fix(search): honor orderBy on unranked search; default newest-first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit memory.search's `orderBy` (and the `me search --order-by` flag) was silently ignored — the unranked (filter-only) path always returned id-ascending (oldest first). Wire it through: search_memory gains an `_order` param applied to the no-ranking arm (whitelisted asc|desc, injection-safe); the space store and RPC handler pass it through. Ranked/hybrid search is unaffected (still score-desc). Default flipped to **desc (newest first)** for filter-only browse, which is the more useful default for a memory store and consistent with ranked search surfacing the top hit first. Pre-production, so no migration needed (fresh schemas pick up the new signature directly). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/memory.ts | 5 ++- .../space/migrate/idempotent/002_search.sql | 10 +++++- packages/engine/space/db.integration.test.ts | 32 +++++++++++++++++++ packages/engine/space/db.ts | 3 +- packages/engine/space/types.ts | 6 ++++ packages/server/rpc/memory/memory.ts | 15 +++++++-- 6 files changed, 65 insertions(+), 6 deletions(-) diff --git a/packages/cli/commands/memory.ts b/packages/cli/commands/memory.ts index 77038a1..a05a7e2 100644 --- a/packages/cli/commands/memory.ts +++ b/packages/cli/commands/memory.ts @@ -230,7 +230,10 @@ function createMemorySearchCommand(): Command { .option("--temporal-within ", "memory must be within (start,end)") .option("--weight-semantic ", "semantic weight (0-1)") .option("--weight-fulltext ", "fulltext weight (0-1)") - .option("--order-by

", "sort direction (asc|desc)") + .option( + "--order-by ", + "filter-only search: order by recency, desc (default, newest first) | asc", + ) .action(async (query: string | undefined, opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); diff --git a/packages/database/space/migrate/idempotent/002_search.sql b/packages/database/space/migrate/idempotent/002_search.sql index 0b6bbbc..d9504e7 100644 --- a/packages/database/space/migrate/idempotent/002_search.sql +++ b/packages/database/space/migrate/idempotent/002_search.sql @@ -16,6 +16,7 @@ create or replace function {{schema}}.search_memory , _temporal_after timestamptz default null , _regexp text default null , _limit bigint default 10 +, _order text default 'desc' -- unranked (filter-only) result order by id: 'desc' (newest first) | 'asc' ) returns table ( id uuid @@ -79,7 +80,14 @@ begin else -- no ranking arm: constant score, typed float8 to match the return column _score = $sql$, (-1)::float8 as score$sql$; - _order_by = $sql$order by m.id$sql$; + -- Order by id — a uuidv7, so creation-time-ordered (and message-time-ordered + -- for the importer's deterministic ids), i.e. a chronological browse. Default + -- desc (newest first). `_order` is whitelisted to asc|desc to keep this + -- interpolation injection-safe. + _order_by = format + ( $sql$order by m.id %s$sql$ + , case when lower(coalesce(_order, 'desc')) = 'asc' then 'asc' else 'desc' end + ); end case; -- ltree diff --git a/packages/engine/space/db.integration.test.ts b/packages/engine/space/db.integration.test.ts index 45f6991..2600ec4 100644 --- a/packages/engine/space/db.integration.test.ts +++ b/packages/engine/space/db.integration.test.ts @@ -104,6 +104,38 @@ test("bm25 search ranks by full-text relevance", async () => { expect(results[0]?.content).toContain("fox"); }); +test("unranked (filter-only) search orders by id, newest-first by default", async () => { + // Explicit, strictly-increasing uuidv7 ids under a dedicated subtree. + const ids = [ + "01900000-0000-7000-8000-000000000001", + "01900000-0000-7000-8000-000000000002", + "01900000-0000-7000-8000-000000000003", + ]; + for (const id of ids) { + await db.createMemory(FULL, { id, tree: "work.ord", content: `c-${id}` }); + } + + // Default → newest id first (desc); results[0] is the high-water entry. + const def = await db.search(FULL, { ltree: "work.ord", limit: 10 }); + expect(def.map((r) => r.id)).toEqual([...ids].reverse()); + + // Explicit asc → oldest first. + const asc = await db.search(FULL, { + ltree: "work.ord", + order: "asc", + limit: 10, + }); + expect(asc.map((r) => r.id)).toEqual(ids); + + // Explicit desc matches the default. + const desc = await db.search(FULL, { + ltree: "work.ord", + order: "desc", + limit: 10, + }); + expect(desc.map((r) => r.id)).toEqual([...ids].reverse()); +}); + test("vector search ranks by embedding similarity", async () => { const near = await db.createMemory(FULL, { tree: "work.v1", diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts index f61a7fa..53cc6a9 100644 --- a/packages/engine/space/db.ts +++ b/packages/engine/space/db.ts @@ -214,7 +214,8 @@ export function spaceStore(sql: Sql, schema: string): SpaceStore { ${o.temporalBefore ?? null}::timestamptz, ${o.temporalAfter ?? null}::timestamptz, ${o.regexp ?? null}, - ${o.limit ?? 10} + ${o.limit ?? 10}, + ${o.order ?? "desc"} )`; return rows.map(mapSearchItem); }, diff --git a/packages/engine/space/types.ts b/packages/engine/space/types.ts index 70ed472..b11f10e 100644 --- a/packages/engine/space/types.ts +++ b/packages/engine/space/types.ts @@ -69,6 +69,12 @@ export interface SearchOptions extends MemoryFilters { /** Max cosine distance (only with `vec`). */ maxVecDist?: number; limit?: number; + /** + * Result order for the **unranked** (filter-only) path: by id (chronological), + * `"desc"` (default, newest first) or `"asc"` (oldest first). Ignored when a + * `bm25`/`vec` query is present — those are ordered by relevance score. + */ + order?: "asc" | "desc"; } export interface HybridSearchOptions extends MemoryFilters { diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 5caf37f..97b3ee6 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -4,8 +4,10 @@ * Adapts the stable memory.* wire protocol onto the space data-plane store * (spaceStore). The wire is unchanged from the legacy engine RPC; the mapping * is handler-local. Lossy by design (see Phase 4C): `createdBy` is always null - * (the space model has no per-memory creator), search `total` is the returned - * row count, and `orderBy` is ignored (ranked search is score-desc only). + * (the space model has no per-memory creator) and search `total` is the returned + * row count. `orderBy` applies to unranked (filter-only) search — chronological + * by id, desc (default, newest first) or asc; ranked/hybrid search ignores it + * (score-desc). */ import { SHARE_NAMESPACE } from "@memory.build/database"; import { generateEmbedding } from "@memory.build/embedding"; @@ -373,7 +375,14 @@ async function memorySearch( ); } else { items = await guard(() => - store.search(treeAccess, { bm25, vec, maxVecDist, limit, ...filters }), + store.search(treeAccess, { + bm25, + vec, + maxVecDist, + limit, + order: params.orderBy ?? undefined, + ...filters, + }), ); } From 2d0cfc5b2a08af08b4a34d51684c2f451f2d5c35 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 12:22:44 +0200 Subject: [PATCH 113/156] docs(todo): track adding behavior tests for all search modes/params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orderBy bug (e9a6eec) slipped through because search tests only assert a match comes back, not that each parameter changes the result. Record a follow-up to cover the full mode/param matrix at the store level + a wire→store plumbing check. Co-Authored-By: Claude Opus 4.8 (1M context) --- TODO.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/TODO.md b/TODO.md index 439835a..72dbba0 100644 --- a/TODO.md +++ b/TODO.md @@ -224,3 +224,27 @@ but unproven at runtime. typecheck errors, and add an end-to-end check that the `me serve` `/rpc` proxy reaches the memory endpoint. Decide whether `packages/web` should be in CI / the root typecheck. + +## Test coverage: every search mode + parameter actually takes effect + +`memory.search`'s `orderBy` was silently ignored for ~the whole pre-release — the +param and the `me search --order-by` flag parsed fine but never reached the SQL +(fixed in `e9a6eec`). Nothing caught it because the search tests only assert +"ranked search returns a match," not "each parameter changes the result." Other +params could be quietly broken the same way. + +- [ ] Add behavior tests (space-store integration level, where the SQL actually + runs) asserting each search **parameter changes the output**, not just that a + query returns rows. Cover the matrix: + - **modes**: bm25-only, vector-only, hybrid (RRF), unranked filter-only. + - **params**: `orderBy` asc/desc (incl. the default), `limit`, + `candidateLimit`, `semanticThreshold`/`maxVecDist`, `weights` + (fulltext/semantic), `tree` (ltree/lquery/ltxtquery), `meta` contains, + `grep`, temporal filters (within/overlaps/contains → before/after). + Each test should construct inputs where the param demonstrably + reorders/filters results (desc vs asc returns the reverse; a tighter + threshold drops a known row; etc.). +- [ ] Add a thin handler/wire-level check (`call("memory.search", …)`) that the + protocol params map onto the store options — so the wire→handler→store + plumbing can't silently drop a field again (the exact gap that hid the + `orderBy` bug: the handler discarded the param before the store ever saw it). From dd28e268e38da84973e112f911eb0746f2f61c4a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 12:42:46 +0200 Subject: [PATCH 114/156] feat(claude-plugin): capture via the import path (Stop/SessionEnd transcript) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the bespoke prompt+final-message hook with full transcript capture that reuses the importer, so live captures and `me … import` are identical by construction — same tree, deterministic ids, and source_* metadata, one memory per message. - importers/index.ts: extract planSession; add importTranscriptFile (parse one transcript → stateless server-derived high-water watermark via a limit-1 newest-first search → write only the delta; full-reconcile fallback for a new session, importer_version bump, lost anchor, or write error). Share DEFAULT_TREE_ROOT/DEFAULT_SESSIONS_NODE_NAME with `me import`. - claude importer: implement parseFile (single transcript → session). - capture.ts: gut to config only (HookConfig + resolveHookConfigFromEnv); add content_mode → fullTranscript; simplify the event shape. - `me claude hook`: handle stop + session-end, reading transcript_path and running importTranscriptFile. hooks.json registers Stop + SessionEnd. - plugin.json: add content_mode userConfig (default "default"; full_transcript also stores reasoning + tool calls/results). - Remove the now-unused resolveProjectSlug (the hook uses SlugRegistry via the import path — one slug code path). Trade-off: an interrupted final turn (no Stop/SessionEnd) isn't captured; per fire reads+parses the transcript + one limit-1 query + a delta batchCreate (no O(N^2), no per-message existence fetch on the hot path). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/cli/me-claude.md | 2 +- .../claude-plugin/.claude-plugin/plugin.json | 7 + packages/claude-plugin/README.md | 38 +-- packages/claude-plugin/hooks/hooks.json | 12 +- packages/cli/claude/capture.test.ts | 294 ++---------------- packages/cli/claude/capture.ts | 246 +++------------ packages/cli/commands/claude.ts | 52 ++-- packages/cli/commands/import.ts | 9 +- packages/cli/importers/claude.ts | 2 + .../cli/importers/import-transcript.test.ts | 143 +++++++++ packages/cli/importers/index.ts | 246 ++++++++++++--- packages/cli/importers/slug.test.ts | 23 +- packages/cli/importers/slug.ts | 19 +- 13 files changed, 500 insertions(+), 593 deletions(-) create mode 100644 packages/cli/importers/import-transcript.test.ts diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index a535971..bad68c4 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -41,7 +41,7 @@ For manual MCP client configuration, see [MCP Integration](../mcp-integration.md ## me claude hook -Invoked by the Claude Code plugin. Reads the event JSON from stdin, resolves config from `CLAUDE_PLUGIN_OPTION_*` env vars, and captures the event as a memory. +Invoked by the Claude Code plugin on `Stop` (each turn) and `SessionEnd`. Reads the `transcript_path` from the event JSON on stdin, resolves config from `CLAUDE_PLUGIN_OPTION_*` env vars (falling back to your `me login` session), and imports the session transcript — the same parse + write as [`me … import`](agent-session-imports.md), incremental so each call only writes messages new since the last. ``` me claude hook --event diff --git a/packages/claude-plugin/.claude-plugin/plugin.json b/packages/claude-plugin/.claude-plugin/plugin.json index 13e3837..ea6c91d 100644 --- a/packages/claude-plugin/.claude-plugin/plugin.json +++ b/packages/claude-plugin/.claude-plugin/plugin.json @@ -36,6 +36,13 @@ "description": "Ltree root under which captures are nested as `..agent_sessions` (the same layout `me ... import` uses, so live and imported sessions share a node per project). Defaults under `share` so your login session can write there; use a `~`-prefixed root to keep captures in your private home instead.", "default": "share.projects", "required": true + }, + "content_mode": { + "type": "string", + "title": "Content mode", + "description": "What to capture per message. `default` stores the user + assistant text (recommended). `full_transcript` also stores reasoning and tool calls/results as their own memories — more complete but much larger/noisier (and may include sensitive tool output).", + "default": "default", + "required": false } } } diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index dd11fd6..9979c33 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -5,9 +5,12 @@ Captures your Claude Code conversations to [Memory Engine](https://memory.build) ## Components - **MCP server** (`me mcp`) — memory tools (search, create, get, update, delete, tree, import, export, etc.) available to the agent during sessions. -- **Hooks**: - - `UserPromptSubmit` captures your prompt as a memory. - - `Stop` captures the agent's final response as a memory. +- **Hooks** — capture the session transcript as memories, one per message: + - `Stop` fires after each turn and imports the session-so-far. + - `SessionEnd` does a final import when the session closes. + - This is the **same parse + write as `me … import`** (incremental: each fire + only writes messages new since the last), so live captures and bulk imports + land in the same place with the same metadata. - **Async, best-effort**. Hooks never block your session; failures log to stderr and exit 0. ## Prerequisites @@ -72,35 +75,28 @@ claude # start a session # → Installed → memory-engine → Configure # → space (OPTIONAL — blank = your active space; pin for project/shared installs) # → api_key (OPTIONAL, sensitive — blank = use your `me login` session) -# → server (default https://api.memory.build) -# → tree_root (default share.projects; captures nest at ..agent_sessions) +# → server (default https://api.memory.build) +# → tree_root (default share.projects; captures nest at ..agent_sessions) +# → content_mode (default | full_transcript — see below) # → values take effect immediately; no restart required ``` -Leave `api_key` blank to use your `me login` session (captures attributed to you); set it to use a dedicated agent key (see above). Leave `space` blank to capture into your active space; pin it for unattended or project-scope installs (a blank space with no active space set means captures are silently skipped). Sensitive values (the api_key) go to your system keychain; non-sensitive values go to the `settings.json` for the scope you installed in. +Leave `api_key` blank to use your `me login` session (captures attributed to you); set it to use a dedicated agent key (see above). Leave `space` blank to capture into your active space; pin it for unattended or project-scope installs (a blank space with no active space set means captures are silently skipped). `content_mode` is `default` (user + assistant text — recommended) or `full_transcript` (also stores reasoning and tool calls/results as their own memories — more complete, but larger/noisier and may include sensitive tool output). Sensitive values (the api_key) go to your system keychain; non-sensitive values go to the `settings.json` for the scope you installed in. ## Verify -After configuring, send a prompt in Claude Code, then check that capture happened: +After a session (each turn's `Stop`, or `SessionEnd`), check that capture happened: ```bash me memory search --tree "share.projects.*" --limit 5 ``` -You should see your recent prompts (and, after the agent finishes, its response). What gets stored: - -- **Tree**: `..agent_sessions` (default root `share.projects`) — same layout as `me … import`, so live + imported sessions share a node per project -- **Metadata** (the same `source_*` schema `me … import` writes, so live + imported sessions are queryable together): - - `type`: `agent_session` - - `source_tool`: `"claude-code"` - - `source_session_id`: Claude Code's session UUID - - `source_message_role`: `user` or `assistant` - - `source_project_slug`: derived from the git `origin` remote (falls back to the cwd basename) - - `source_cwd`: working directory when the hook fired - - `source_git_repo`: the git remote URL (when the cwd is in a repo) - - `content_mode`: `default` - - `me_version`: the `me` CLI version that created the memory -- **Temporal**: ISO timestamp of hook invocation +Capture is the **same path as `me … import`** — one memory per message, with the +identical layout and metadata, so live and imported sessions interleave cleanly: + +- **Tree**: `..agent_sessions` (default root `share.projects`) — one node per project. +- **Metadata**: the importer's `source_*` schema — `type: agent_session`, `source_tool: "claude"`, `source_session_id`, `source_message_id`, `source_message_role` (`user`/`assistant`), `source_project_slug` (from the git `origin` remote, else cwd basename), `content_mode`, `importer_version`, and (when available) `source_cwd` / `source_git_repo` / `source_model` / … See the full table in [agent session imports](https://docs.memory.build/cli/agent-session-imports). +- **Temporal**: each memory's `start` is the **message** timestamp. ## Multi-scope / multi-engine diff --git a/packages/claude-plugin/hooks/hooks.json b/packages/claude-plugin/hooks/hooks.json index bf2f955..196e65c 100644 --- a/packages/claude-plugin/hooks/hooks.json +++ b/packages/claude-plugin/hooks/hooks.json @@ -1,26 +1,26 @@ { "description": "Memory Engine capture hooks", "hooks": { - "UserPromptSubmit": [ + "Stop": [ { "hooks": [ { "type": "command", - "command": "me claude hook --event user-prompt-submit", + "command": "me claude hook --event stop", "async": true, - "timeout": 30 + "timeout": 60 } ] } ], - "Stop": [ + "SessionEnd": [ { "hooks": [ { "type": "command", - "command": "me claude hook --event stop", + "command": "me claude hook --event session-end", "async": true, - "timeout": 30 + "timeout": 60 } ] } diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index cbc55d4..50106e1 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -1,269 +1,44 @@ /** - * Unit tests for Claude Code hook capture logic. + * Unit tests for Claude Code hook config resolution. + * + * Capture itself is the import path (importTranscriptFile, tested in + * packages/cli/importers). This file only covers resolveHookConfigFromEnv — + * bearer/space/server/tree-root/content-mode resolution + session fallback. */ import { describe, expect, test } from "bun:test"; -import type { MemoryClient } from "@memory.build/client"; -import { - buildMeta, - captureHookEvent, - extractContent, - type HookConfig, - type HookEvent, - messageRoleForEvent, - resolveHookConfigFromEnv, -} from "./capture.ts"; - -const BASE_EVENT = { - session_id: "sess-abc", - cwd: "/tmp/myproj", - hook_event_name: "UserPromptSubmit", - transcript_path: "/tmp/transcript.jsonl", -}; - -const CONFIG: HookConfig = { - server: "https://api.example.com", - token: "me.lookupid12345678.secret", - space: "eng123", - treeRoot: "share.projects", -}; - -// ============================================================================= -// extractContent -// ============================================================================= - -describe("extractContent", () => { - test("returns prompt for user-prompt-submit", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: "hello world" }; - expect(extractContent(event, "user-prompt-submit")).toBe("hello world"); - }); - - test("returns null for empty prompt", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: "" }; - expect(extractContent(event, "user-prompt-submit")).toBeNull(); - }); - - test("returns null for whitespace-only prompt", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: " \n\t " }; - expect(extractContent(event, "user-prompt-submit")).toBeNull(); - }); - - test("returns last_assistant_message for stop", () => { - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: "final response", - }; - expect(extractContent(event, "stop")).toBe("final response"); - }); - - test("returns null for null last_assistant_message", () => { - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: null, - }; - expect(extractContent(event, "stop")).toBeNull(); - }); - - test("returns null for missing last_assistant_message", () => { - const event: HookEvent = { ...BASE_EVENT }; - expect(extractContent(event, "stop")).toBeNull(); - }); - - test("preserves internal whitespace in content", () => { - const event: HookEvent = { - ...BASE_EVENT, - prompt: "line1\n\nline2\n", - }; - expect(extractContent(event, "user-prompt-submit")).toBe( - "line1\n\nline2\n", - ); - }); -}); - -// ============================================================================= -// messageRoleForEvent -// ============================================================================= - -describe("messageRoleForEvent", () => { - test("maps user-prompt-submit to user", () => { - expect(messageRoleForEvent("user-prompt-submit")).toBe("user"); - }); - - test("maps stop to assistant", () => { - expect(messageRoleForEvent("stop")).toBe("assistant"); - }); -}); - -// (Project-slug derivation lives in importers/slug.ts — resolveProjectSlug — -// and is covered by slug.test.ts; the hook just consumes it.) - -// ============================================================================= -// buildMeta — aligned with the import tool's source_* schema -// ============================================================================= - -describe("buildMeta", () => { - test("builds the source_* metadata for a user prompt", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: "hi" }; - const meta = buildMeta(event, "user-prompt-submit", "myproject"); - - expect(meta.type).toBe("agent_session"); - expect(meta.source_tool).toBe("claude-code"); - expect(meta.source_session_id).toBe("sess-abc"); - expect(meta.source_message_role).toBe("user"); - expect(meta.source_project_slug).toBe("myproject"); - expect(meta.source_cwd).toBe("/tmp/myproj"); - expect(meta.content_mode).toBe("default"); - expect(typeof meta.me_version).toBe("string"); - // No git remote passed → no source_git_repo. - expect(meta.source_git_repo).toBeUndefined(); - }); - - test("uses the assistant role for a stop event", () => { - const event: HookEvent = { ...BASE_EVENT, last_assistant_message: "done" }; - const meta = buildMeta(event, "stop", "proj"); - expect(meta.type).toBe("agent_session"); - expect(meta.source_message_role).toBe("assistant"); - }); - - test("includes source_git_repo when a remote is provided", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: "hi" }; - const meta = buildMeta( - event, - "user-prompt-submit", - "proj", - "git@github.com:org/repo.git", - ); - expect(meta.source_git_repo).toBe("git@github.com:org/repo.git"); - }); -}); - -// ============================================================================= -// captureHookEvent -// ============================================================================= - -/** Build a mock MemoryClient that records the last memory.create call. */ -function mockClient(): { - client: MemoryClient; - calls: Array>; -} { - const calls: Array> = []; - const client = { - memory: { - create: async (params: Record) => { - calls.push(params); - return { id: "01960000-0000-7000-8000-000000000000" }; - }, - }, - } as unknown as MemoryClient; - return { client, calls }; -} - -describe("captureHookEvent", () => { - test("skips empty content with no API call", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { ...BASE_EVENT, prompt: " " }; - - const result = await captureHookEvent(event, "user-prompt-submit", CONFIG, { - client, - }); - - expect(result.status).toBe("skipped"); - expect(calls).toHaveLength(0); - }); - - test("captures user prompt with correct tree + meta", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { ...BASE_EVENT, prompt: "hello" }; - const now = new Date("2026-04-23T10:00:00Z"); - - const result = await captureHookEvent(event, "user-prompt-submit", CONFIG, { - client, - now: () => now, - }); - - expect(result.status).toBe("captured"); - expect(result.memoryId).toBe("01960000-0000-7000-8000-000000000000"); - expect(calls).toHaveLength(1); - const [call] = calls as [Record]; - expect(call.content).toBe("hello"); - expect(call.temporal).toEqual({ start: "2026-04-23T10:00:00.000Z" }); - const meta = call.meta as Record; - // Nested as ..agent_sessions (project derived from cwd). - expect(call.tree).toBe( - `share.projects.${meta.source_project_slug}.agent_sessions`, - ); - expect(meta.type).toBe("agent_session"); - expect(meta.source_message_role).toBe("user"); - expect(meta.source_session_id).toBe("sess-abc"); - expect(meta.source_tool).toBe("claude-code"); - }); - - test("captures stop event with the assistant role", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: "goodbye", - }; - - const result = await captureHookEvent(event, "stop", CONFIG, { client }); - - expect(result.status).toBe("captured"); - expect(calls).toHaveLength(1); - const [call] = calls as [Record]; - expect(call.content).toBe("goodbye"); - const meta = call.meta as Record; - expect(meta.type).toBe("agent_session"); - expect(meta.source_message_role).toBe("assistant"); - }); - - test("nests under a custom treeRoot from config", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { ...BASE_EVENT, prompt: "x" }; - const cfg: HookConfig = { - ...CONFIG, - treeRoot: "~.work", - }; - - await captureHookEvent(event, "user-prompt-submit", cfg, { client }); - - const [call] = calls as [Record]; - const meta = call.meta as Record; - expect(call.tree).toBe(`~.work.${meta.source_project_slug}.agent_sessions`); - }); -}); - -// ============================================================================= -// resolveHookConfigFromEnv -// ============================================================================= +import { type HookConfig, resolveHookConfigFromEnv } from "./capture.ts"; describe("resolveHookConfigFromEnv", () => { test("returns null when no api_key and no session fallback", () => { - const cfg = resolveHookConfigFromEnv({}); - expect(cfg).toBeNull(); + expect(resolveHookConfigFromEnv({})).toBeNull(); }); - test("returns null when space is missing (keys are global)", () => { - const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", - }); - expect(cfg).toBeNull(); + test("returns null when space is missing", () => { + expect( + resolveHookConfigFromEnv({ + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + }), + ).toBeNull(); }); - test("returns config when api_key and space are present", () => { + test("resolves full config from plugin env", () => { const cfg = resolveHookConfigFromEnv({ CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", CLAUDE_PLUGIN_OPTION_SERVER: "https://api.example.com", - CLAUDE_PLUGIN_OPTION_TREE_ROOT: "my.root", + CLAUDE_PLUGIN_OPTION_TREE_ROOT: "share.work", + CLAUDE_PLUGIN_OPTION_CONTENT_MODE: "full_transcript", }); expect(cfg).toEqual({ token: "me.lookupid12345678.secret", space: "eng123def456", server: "https://api.example.com", - treeRoot: "my.root", - }); + treeRoot: "share.work", + fullTranscript: true, + } satisfies HookConfig); }); - test("falls back to default server and tree_root", () => { + test("defaults: server, tree root, content mode (default = not full)", () => { const cfg = resolveHookConfigFromEnv({ CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", @@ -273,7 +48,17 @@ describe("resolveHookConfigFromEnv", () => { space: "eng123def456", server: "https://api.memory.build", treeRoot: "share.projects", + fullTranscript: false, + } satisfies HookConfig); + }); + + test("content_mode=default → fullTranscript false", () => { + const cfg = resolveHookConfigFromEnv({ + CLAUDE_PLUGIN_OPTION_API_KEY: "k", + CLAUDE_PLUGIN_OPTION_SPACE: "s", + CLAUDE_PLUGIN_OPTION_CONTENT_MODE: "default", }); + expect(cfg?.fullTranscript).toBe(false); }); test("falls back to the login session when api_key is blank", () => { @@ -281,12 +66,8 @@ describe("resolveHookConfigFromEnv", () => { { CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456" }, { sessionToken: "sess-token", server: "https://api.example.com" }, ); - expect(cfg).toEqual({ - token: "sess-token", - space: "eng123def456", - server: "https://api.example.com", - treeRoot: "share.projects", - }); + expect(cfg?.token).toBe("sess-token"); + expect(cfg?.server).toBe("https://api.example.com"); }); test("treats an unsubstituted ${...} placeholder as blank (uses session)", () => { @@ -318,15 +99,4 @@ describe("resolveHookConfigFromEnv", () => { ); expect(cfg?.token).toBe("me.lookupid12345678.secret"); }); - - test("treats empty string as missing (falls back to default)", () => { - const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", - CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", - CLAUDE_PLUGIN_OPTION_SERVER: "", - CLAUDE_PLUGIN_OPTION_TREE_ROOT: "", - }); - expect(cfg?.server).toBe("https://api.memory.build"); - expect(cfg?.treeRoot).toBe("share.projects"); - }); }); diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index 0bf79c0..ee941e5 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -1,146 +1,57 @@ /** - * Claude Code hook event parsing and memory capture. + * Claude Code capture hook — config resolution + event shape. * - * Pure functions for event parsing and metadata construction are testable in - * isolation. The `captureHookEvent` entry point handles memory creation via the - * memory client. The project label is derived by the shared `resolveProjectSlug` - * (the same logic the import tool uses) so live + imported sessions for a repo - * share one project node. + * Capture itself is the import path: the hook reads the session transcript and + * runs it through `importTranscriptFile` (packages/cli/importers), so live + * captures and `me import` produce identical memories (tree, ids, `source_*` + * metadata). This module only resolves the runtime config (bearer + space + + * tree root + content mode) and types the slice of the hook event payload we + * read. The orchestration lives in `commands/claude.ts` (`me claude hook`). */ +import { + DEFAULT_SESSIONS_NODE_NAME, + DEFAULT_TREE_ROOT, +} from "../importers/index.ts"; -import { CLIENT_VERSION } from "../../../version"; -import { createMemoryClient, type MemoryClient } from "../client.ts"; -import { resolveProjectSlug } from "../importers/slug.ts"; +export const DEFAULT_SERVER = "https://api.memory.build"; + +/** Per-project sessions leaf, shared with `me import`. */ +export const SESSIONS_NODE = DEFAULT_SESSIONS_NODE_NAME; -// ============================================================================= -// Hook config (derived at runtime from CLAUDE_PLUGIN_OPTION_* env vars) -// ============================================================================= +/** + * Hook events the plugin registers. Both drive a full transcript import (Stop + * per turn; SessionEnd as a final flush) — idempotent, so re-importing is a + * no-op for already-captured messages. + */ +export const HOOK_EVENT_NAMES = ["stop", "session-end"] as const; +export type HookEventName = (typeof HOOK_EVENT_NAMES)[number]; + +/** The slice of a Claude Code hook event payload the capture hook reads. */ +export interface HookEvent { + session_id?: string; + cwd?: string; + /** Path to the session transcript JSONL (present on Stop / SessionEnd). */ + transcript_path?: string; + hook_event_name?: string; +} +/** Resolved hook config: where + how to write captured memories. */ export interface HookConfig { /** Memory Engine server URL. */ server: string; /** * Bearer for the memory endpoint: the plugin's api key (sensitive userConfig) - * when set, else the user's `me login` session token. Both authenticate the - * memory endpoint. + * when set, else the user's `me login` session token. */ token: string; /** Active space slug (X-Me-Space). */ space: string; - /** - * Tree root under which captures are nested as - * `..` — the same shape the import tool - * writes, so live captures and imported sessions share one node per project. - */ + /** Tree root; captures nest as `..agent_sessions`. */ treeRoot: string; + /** content_mode=full_transcript → also store reasoning + tool calls/results. */ + fullTranscript: boolean; } -// ============================================================================= -// Event types -// ============================================================================= - -export type HookEventName = "user-prompt-submit" | "stop"; - -export const HOOK_EVENT_NAMES: HookEventName[] = ["user-prompt-submit", "stop"]; - -/** Fields common to all Claude Code hook events. */ -interface HookEventBase { - session_id: string; - transcript_path?: string; - cwd: string; - hook_event_name?: string; -} - -export interface UserPromptSubmitEvent extends HookEventBase { - prompt: string; -} - -export interface StopEvent extends HookEventBase { - last_assistant_message?: string | null; - stop_hook_active?: boolean; -} - -export type HookEvent = UserPromptSubmitEvent | StopEvent; - -// ============================================================================= -// Pure helpers (testable) -// ============================================================================= - -/** - * Extract the memory content from a hook event. - * - * Returns null if the event has no content to capture. - */ -export function extractContent( - event: HookEvent, - eventName: HookEventName, -): string | null { - if (eventName === "user-prompt-submit") { - const prompt = (event as UserPromptSubmitEvent).prompt; - if (typeof prompt !== "string") return null; - const trimmed = prompt.trim(); - return trimmed.length > 0 ? prompt : null; - } - - if (eventName === "stop") { - const msg = (event as StopEvent).last_assistant_message; - if (typeof msg !== "string") return null; - const trimmed = msg.trim(); - return trimmed.length > 0 ? msg : null; - } - - return null; -} - -/** Map an event name to the message role (`source_message_role`). */ -export function messageRoleForEvent( - eventName: HookEventName, -): "user" | "assistant" { - return eventName === "user-prompt-submit" ? "user" : "assistant"; -} - -/** - * Build the metadata for a captured memory. Uses the same `source_*` schema as - * the import tool (see buildMeta in packages/cli/importers/index.ts) so live - * captures and imported sessions co-located in a project node are uniformly - * queryable. `type` is the constant `agent_session`; the prompt/response - * distinction lives in `source_message_role`. - */ -export function buildMeta( - event: HookEvent, - eventName: HookEventName, - project: string, - gitRepo?: string, -): Record { - const meta: Record = { - type: "agent_session", - source_tool: "claude-code", - source_session_id: event.session_id, - source_message_role: messageRoleForEvent(eventName), - source_project_slug: project, - source_cwd: event.cwd, - content_mode: "default", - me_version: CLIENT_VERSION, - }; - if (gitRepo) meta.source_git_repo = gitRepo; - return meta; -} - -// ============================================================================= -// Config resolution from environment -// ============================================================================= - -export const DEFAULT_SERVER = "https://api.memory.build"; -// Captures nest as `..`, identical to the -// import tool's default layout (see DEFAULT_TREE_ROOT / DEFAULT_SESSIONS_NODE_NAME -// in packages/cli/commands/import.ts), so live + imported sessions for a project -// land in the same node (distinguished by meta.source). Under `share` so a -// session-authenticated user (owner@share, not arbitrary top-level paths) can -// write here. -export const DEFAULT_TREE_ROOT = "share.projects"; -// Fixed per-project leaf, matching the import tool's --sessions-node-name default. -const SESSIONS_NODE = "agent_sessions"; - /** Credentials the hook falls back to when the plugin's api_key is unset. */ export interface HookFallbackCreds { apiKey?: string; @@ -159,14 +70,11 @@ function blank(v: string | undefined): boolean { } /** - * Resolve the hook config. The bearer is the plugin's `api_key` (sensitive - * userConfig, delivered via `CLAUDE_PLUGIN_OPTION_API_KEY`) when set; otherwise - * it falls back to the user's `me login` session (passed in via `creds`, so this - * function stays pure/testable). The space comes from the plugin config, else the - * caller's active space. Returns null when no bearer or no space is available. - * - * Claude Code delivers `sensitive: true` userConfig values (like api_key) through - * the same env var mechanism as non-sensitive ones. + * Resolve the hook config. The bearer is the plugin's `api_key` + * (`CLAUDE_PLUGIN_OPTION_API_KEY`) when set; otherwise it falls back to the + * user's `me login` session (passed in via `creds`, so this stays pure/testable). + * The space comes from the plugin config, else the caller's active space. + * Returns null when no bearer or no space is available. */ export function resolveHookConfigFromEnv( env: NodeJS.ProcessEnv = process.env, @@ -179,8 +87,7 @@ export function resolveHookConfigFromEnv( const token = pluginKey ?? creds.apiKey ?? creds.sessionToken; if (!token) return null; - // Space: plugin config, else the active space. Required either way (api keys - // are global, and a session still needs a target space). + // Space: plugin config, else the active space. Required either way. const space = blank(env.CLAUDE_PLUGIN_OPTION_SPACE) ? creds.activeSpace : env.CLAUDE_PLUGIN_OPTION_SPACE; @@ -190,68 +97,13 @@ export function resolveHookConfigFromEnv( ? (creds.server ?? DEFAULT_SERVER) : (env.CLAUDE_PLUGIN_OPTION_SERVER as string); - return { - token, - space, - server, - treeRoot: env.CLAUDE_PLUGIN_OPTION_TREE_ROOT || DEFAULT_TREE_ROOT, - }; -} - -// ============================================================================= -// Capture entry point -// ============================================================================= - -export interface CaptureResult { - status: "captured" | "skipped"; - reason?: string; - memoryId?: string; -} - -export interface CaptureOptions { - /** Override the client (for tests). */ - client?: MemoryClient; - /** Override timestamp (for deterministic tests). */ - now?: () => Date; -} - -/** - * Capture a hook event as a memory. - * - * Returns immediately if there's no content to capture. Otherwise creates a - * memory under `..agent_sessions` with metadata. - */ -export async function captureHookEvent( - event: HookEvent, - eventName: HookEventName, - config: HookConfig, - opts: CaptureOptions = {}, -): Promise { - const content = extractContent(event, eventName); - if (content === null) { - return { status: "skipped", reason: "empty content" }; - } - - const { slug: project, gitRemote } = await resolveProjectSlug(event.cwd); - const meta = buildMeta(event, eventName, project, gitRemote); - const now = (opts.now ?? (() => new Date()))(); - - const client = - opts.client ?? - createMemoryClient({ - url: config.server, - token: config.token, - space: config.space, - }); + const treeRoot = blank(env.CLAUDE_PLUGIN_OPTION_TREE_ROOT) + ? DEFAULT_TREE_ROOT + : (env.CLAUDE_PLUGIN_OPTION_TREE_ROOT as string); - // Nest by project under the configured root, with a fixed sessions leaf — - // `..` — matching the import tool's layout. - const result = await client.memory.create({ - content, - tree: `${config.treeRoot}.${project}.${SESSIONS_NODE}`, - meta, - temporal: { start: now.toISOString() }, - }); + const fullTranscript = + (env.CLAUDE_PLUGIN_OPTION_CONTENT_MODE ?? "").toLowerCase() === + "full_transcript"; - return { status: "captured", memoryId: result.id }; + return { server, token, space, treeRoot, fullTranscript }; } diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 27f8447..bd005bc 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -21,14 +21,16 @@ */ import { Command, InvalidArgumentError } from "commander"; import { - captureHookEvent, HOOK_EVENT_NAMES, type HookEvent, type HookEventName, resolveHookConfigFromEnv, + SESSIONS_NODE, } from "../claude/capture.ts"; +import { createMemoryClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { claudeImporter } from "../importers/claude.ts"; +import { importTranscriptFile } from "../importers/index.ts"; import { type AgentInstallOptions, runAgentMcpInstall, @@ -89,12 +91,14 @@ function createClaudeInstallCommand(): Command { } /** - * me claude hook — invoked by the Claude Code plugin to capture events as - * memories. + * me claude hook — invoked by the Claude Code plugin on Stop / SessionEnd to + * capture the session. * - * Reads the event JSON from stdin, resolves config from the CLAUDE_PLUGIN_OPTION_* - * env vars Claude Code exports for the plugin (falling back to the `me login` - * session when no api_key is configured), and creates a memory. + * Reads the event JSON from stdin for the `transcript_path`, resolves config + * from the CLAUDE_PLUGIN_OPTION_* env vars (falling back to the `me login` + * session when no api_key is configured), and runs the transcript through + * `importTranscriptFile` — the same parse + write as `me import`, incremental so + * each call only writes messages new since the last. * * Best-effort: logs failures to stderr but always exits 0 so that a hook * failure never blocks a Claude Code session. @@ -129,35 +133,39 @@ function createClaudeHookCommand(): Command { process.exit(0); } - // Read stdin - let input: string; + // Read + parse the event JSON from stdin for the transcript path. + let event: HookEvent; try { - input = await Bun.stdin.text(); + event = JSON.parse(await Bun.stdin.text()) as HookEvent; } catch (error) { console.error( - `[memory-engine] failed to read stdin: ${error instanceof Error ? error.message : String(error)}`, + `[memory-engine] failed to read/parse event JSON: ${error instanceof Error ? error.message : String(error)}`, ); process.exit(0); } - // Parse JSON - let event: HookEvent; - try { - event = JSON.parse(input) as HookEvent; - } catch (error) { + const transcriptPath = event.transcript_path; + if (!transcriptPath) { console.error( - `[memory-engine] failed to parse event JSON: ${error instanceof Error ? error.message : String(error)}`, + `[memory-engine] ${eventName}: no transcript_path in event payload`, ); process.exit(0); } - // Capture + // Import the transcript (incremental; same path as `me import`). try { - const result = await captureHookEvent(event, eventName, config); - if (result.status === "skipped") { - // Silent skip — no stderr output for empty content - process.exit(0); - } + const client = createMemoryClient({ + url: config.server, + token: config.token, + space: config.space, + }); + await importTranscriptFile(client, claudeImporter, transcriptPath, { + treeRoot: config.treeRoot, + sessionsNodeName: SESSIONS_NODE, + fullTranscript: config.fullTranscript, + dryRun: false, + verbose: false, + }); } catch (error) { console.error( `[memory-engine] ${eventName} capture failed: ${error instanceof Error ? error.message : String(error)}`, diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index 8efbb2b..89a20eb 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -27,6 +27,8 @@ import { Command } from "commander"; import { resolveCredentials } from "../credentials.ts"; import { createProgressReporter, + DEFAULT_SESSIONS_NODE_NAME, + DEFAULT_TREE_ROOT, type Importer, type ImportResult, runImport, @@ -41,11 +43,8 @@ import { requireSpace, } from "../util.ts"; -// Under `share` so a session-authenticated user can write the import (they hold -// owner@share, not access to arbitrary top-level paths). Override with -// --tree-root for a different destination you have write access to. -const DEFAULT_TREE_ROOT = "share.projects"; -const DEFAULT_SESSIONS_NODE_NAME = "agent_sessions"; +// Default capture layout (share.projects..agent_sessions) lives in the +// importers module so `me import` and the Claude Code hook share one source. // Lenient user-facing tree-path input (matches the protocol's treePathSchema): // labels [A-Za-z0-9_-], `.` or `/` separators, optional leading `~` (home). The // server normalizes + authoritatively validates; this is a fast pre-check so a diff --git a/packages/cli/importers/claude.ts b/packages/cli/importers/claude.ts index 6658dca..96d5a5a 100644 --- a/packages/cli/importers/claude.ts +++ b/packages/cli/importers/claude.ts @@ -77,6 +77,8 @@ export const claudeImporter: Importer = { tool: "claude", defaultSource: DEFAULT_SOURCE, discoverSessions, + // Single-file parse for the live capture hook (importTranscriptFile). + parseFile: parseSessionFile, }; async function* discoverSessions( diff --git a/packages/cli/importers/import-transcript.test.ts b/packages/cli/importers/import-transcript.test.ts new file mode 100644 index 0000000..02557f2 --- /dev/null +++ b/packages/cli/importers/import-transcript.test.ts @@ -0,0 +1,143 @@ +/** + * Unit tests for importTranscriptFile — the live-capture (Claude hook) path. + * + * Uses a fake importer (parseFile returns a synthetic session) + an in-memory + * mock client that round-trips meta through the real buildMeta, so the + * watermark / incremental-delta / reconcile-fallback logic is exercised without + * a database. + */ +import { describe, expect, test } from "bun:test"; +import type { MemoryClient } from "../client.ts"; +import { + type Importer, + importTranscriptFile, + type WriteOptions, +} from "./index.ts"; +import type { ConversationMessage, ImportedSession } from "./types.ts"; + +const WRITE: WriteOptions = { + treeRoot: "share.projects", + sessionsNodeName: "agent_sessions", + fullTranscript: false, + dryRun: false, + verbose: false, +}; + +/** A mock engine backed by an in-memory id→meta store, mimicking the server. */ +function mockEngine() { + const store = new Map< + string, + { id: string; meta: Record } + >(); + const client = { + memory: { + // Filter by source_session_id, order by id desc (server default), slice to limit. + search: async (p: { meta?: Record; limit?: number }) => { + const sid = p.meta?.source_session_id; + const all = [...store.values()] + .filter((m) => m.meta.source_session_id === sid) + .sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)); + const limit = p.limit ?? 10; + return { results: all.slice(0, limit), total: all.length, limit }; + }, + batchCreate: async (p: { + memories: Array<{ id: string; meta: Record }>; + }) => { + const ids: string[] = []; + for (const m of p.memories) { + if (store.has(m.id)) throw new Error(`duplicate id ${m.id}`); + store.set(m.id, { id: m.id, meta: m.meta }); + ids.push(m.id); + } + return { ids }; + }, + }, + } as unknown as MemoryClient; + return { client, store }; +} + +/** An importer whose parseFile returns a fixed session (or null). */ +function importerFor(session: ImportedSession | null): Importer { + return { + tool: "claude", + defaultSource: "", + // biome-ignore lint/correctness/useYield: empty stub generator + discoverSessions: async function* () {}, + parseFile: async () => session, + }; +} + +/** Build a session whose messages have strictly-increasing timestamps. */ +function session(messageIds: string[]): ImportedSession { + const messages: ConversationMessage[] = messageIds.map((id, i) => ({ + messageId: id, + timestamp: new Date(Date.UTC(2026, 0, 1, 0, 0, i)).toISOString(), + role: i % 2 === 0 ? "user" : "assistant", + blocks: [{ kind: "text", text: `message ${id}` }], + })); + return { + tool: "claude", + sessionId: "sess-1", + cwd: "/tmp/nonexistent-import-transcript-test/myproj", + sourceFile: "/tmp/transcript.jsonl", + startedAt: messages[0]?.timestamp ?? "2026-01-01T00:00:00.000Z", + endedAt: messages.at(-1)?.timestamp ?? "2026-01-01T00:00:00.000Z", + sourceModifiedAt: "2026-01-01T00:00:00.000Z", + messages, + }; +} + +describe("importTranscriptFile", () => { + test("returns null when the file has no session", async () => { + const { client, store } = mockEngine(); + expect( + await importTranscriptFile(client, importerFor(null), "/x.jsonl", WRITE), + ).toBeNull(); + expect(store.size).toBe(0); + }); + + test("first import writes every message (reconcile path)", async () => { + const { client, store } = mockEngine(); + const out = await importTranscriptFile( + client, + importerFor(session(["a", "b", "c"])), + "/x.jsonl", + WRITE, + ); + expect(out?.inserted).toBe(3); + expect(store.size).toBe(3); + }); + + test("re-importing the same transcript is a no-op (watermark fast path)", async () => { + const { client, store } = mockEngine(); + const imp = () => + importTranscriptFile( + client, + importerFor(session(["a", "b", "c"])), + "/x.jsonl", + WRITE, + ); + await imp(); + const again = await imp(); + expect(again?.inserted).toBe(0); + expect(store.size).toBe(3); + }); + + test("only messages new since the watermark are written", async () => { + const { client, store } = mockEngine(); + await importTranscriptFile( + client, + importerFor(session(["a", "b", "c"])), + "/x.jsonl", + WRITE, + ); + const out = await importTranscriptFile( + client, + importerFor(session(["a", "b", "c", "d"])), + "/x.jsonl", + WRITE, + ); + expect(out?.inserted).toBe(1); // just "d" + expect(store.size).toBe(4); + }); +}); diff --git a/packages/cli/importers/index.ts b/packages/cli/importers/index.ts index 4257da1..3b8da99 100644 --- a/packages/cli/importers/index.ts +++ b/packages/cli/importers/index.ts @@ -46,6 +46,16 @@ export const IMPORTER_VERSION = "1"; */ const SEARCH_PAGE_LIMIT = 1000; +/** + * Default capture layout, shared by `me import` and the Claude Code capture + * hook so live + imported sessions land in the same place: + * `..`. Under + * `share` so a session-authenticated user (owner@share, not arbitrary top-level + * paths) can write there. + */ +export const DEFAULT_TREE_ROOT = "share.projects"; +export const DEFAULT_SESSIONS_NODE_NAME = "agent_sessions"; + /** An importer's discovery interface — yields normalized sessions. */ export interface Importer { tool: ImportedSession["tool"]; @@ -55,6 +65,12 @@ export interface Importer { stats: ImporterStats, progress?: ProgressReporter, ): AsyncIterable; + /** + * Parse a single transcript file into one session (or null if empty / no + * messages). Used by the live capture hook (`importTranscriptFile`); only the + * Claude importer implements it for now. + */ + parseFile?(path: string): Promise; } /** Result of the orchestration pass. */ @@ -162,6 +178,122 @@ export async function runImport( }; } +/** + * Import a single transcript file — the live-capture path used by the Claude + * Code hook. Reuses the same parse + render + write as `me import`, so live + * captures and bulk imports produce identical memories (tree, ids, `source_*` + * metadata). + * + * Incremental + stateless: it asks the server for the session's high-water + * message (`searchSessionHighWater` — one `limit 1`, newest-first search) and + * writes only the messages after it. Falls back to the full reconcile + * (`writeSession`) for a new session, an `importer_version` bump, a lost anchor + * (compaction/reorder), or any fast-path write error — so correctness never + * depends on the optimization. Returns null when the file has no session. + */ +export async function importTranscriptFile( + engine: MemoryClient, + importer: Importer, + filePath: string, + options: WriteOptions, +): Promise { + if (!importer.parseFile) { + throw new Error( + `importer '${importer.tool}' does not support single-file parsing`, + ); + } + const session = await importer.parseFile(filePath); + if (!session) return null; + + const { slug, gitRoot, gitRemote } = await new SlugRegistry().resolve( + session.cwd, + ); + const tree = `${options.treeRoot}.${slug}.${options.sessionsNodeName}`; + const title = synthesizeTitle(session); + + const hw = await searchSessionHighWater( + engine, + tree, + session.tool, + session.sessionId, + ); + if (hw && hw.importerVersion === IMPORTER_VERSION) { + const plan = planSession(session, tree, slug, gitRoot, gitRemote, options); + const idx = plan.planned.findIndex( + (p) => p.message.messageId === hw.messageId, + ); + if (idx !== -1) { + const delta = plan.planned.slice(idx + 1); + const outcome: SessionOutcome = { + sessionId: session.sessionId, + title, + tree, + sourceFile: session.sourceFile, + inserted: 0, + updated: 0, + skipped: plan.skipped, + failed: plan.failed, + errors: [...plan.errors], + }; + if (delta.length === 0) return outcome; + if (options.dryRun) { + outcome.inserted += delta.length; + return outcome; + } + try { + const { insertedIds, errors } = await batchCreateChunked( + engine, + delta.map((d) => d.payload), + ); + if (errors.length === 0) { + outcome.inserted += insertedIds.length; + return outcome; + } + // Partial failure (e.g. an already-present id from a non-monotonic + // transcript) → fall through to the full reconcile for correctness. + } catch { + // Unexpected write error → reconcile. + } + } + } + + return writeSession( + engine, + session, + title, + tree, + slug, + gitRoot, + gitRemote, + options, + ); +} + +/** + * The session's high-water message: the latest already-imported message for + * (tool, sessionId) under `tree`. One `memory.search` with `limit: 1` — unranked + * search defaults to newest-first (by id, which encodes the message timestamp), + * so results[0] is the most recent. Null when nothing is imported yet. + */ +async function searchSessionHighWater( + engine: MemoryClient, + tree: string, + tool: ImportedSession["tool"], + sessionId: string, +): Promise<{ messageId: string; importerVersion?: string } | null> { + const res = await engine.memory.search({ + tree, + meta: { source_tool: tool, source_session_id: sessionId }, + limit: 1, + }); + const top = res.results[0]; + if (!top) return null; + const messageId = top.meta.source_message_id; + if (typeof messageId !== "string") return null; + const v = top.meta.importer_version; + return { messageId, importerVersion: typeof v === "string" ? v : undefined }; +} + /** * Write all messages for one session. * @@ -175,49 +307,52 @@ export async function runImport( * 3. Issue one `memory.batchCreate` (in chunks of 1000) for inserts; * updates are issued one at a time (rare path). */ -async function writeSession( - engine: MemoryClient, +/** One planned message write (post-render, pre-dedup/diff). */ +interface PlannedMessage { + message: ConversationMessage; + memoryId: string; + payload: MemoryCreateParams; +} + +/** Result of planning a session's writes (rendered, deduped). */ +interface PlanResult { + planned: PlannedMessage[]; + skipped: number; + failed: number; + errors: Array<{ messageId: string; error: string }>; +} + +/** + * Render + dedup a session's messages into write payloads (no RPCs). Skips + * messages that render empty under the chosen mode, records bad timestamps as + * failures, and collapses events sharing a deterministic id (resume/replay + * artefacts) so the batch can't trip the unique constraint server-side. + */ +function planSession( session: ImportedSession, - title: string, tree: string, projectSlug: string, gitRoot: string | undefined, gitRemote: string | undefined, options: WriteOptions, -): Promise { - const outcome: SessionOutcome = { - sessionId: session.sessionId, - title, - tree, - sourceFile: session.sourceFile, - inserted: 0, - updated: 0, - skipped: 0, - failed: 0, - errors: [], - }; - - // Build the per-message write payloads up front so we can plan the - // batch in one pass and skip any messages that render to empty - // content under the chosen mode. - const planned: Array<{ - message: ConversationMessage; - memoryId: string; - payload: MemoryCreateParams; - }> = []; +): PlanResult { + const planned: PlannedMessage[] = []; + let skipped = 0; + let failed = 0; + const errors: Array<{ messageId: string; error: string }> = []; for (const message of session.messages) { const content = renderMessageContent(message, { fullTranscript: options.fullTranscript, }); if (content === null) { - outcome.skipped++; + skipped++; continue; } const timestampMs = Number(Date.parse(message.timestamp)); if (Number.isNaN(timestampMs)) { - outcome.failed++; - outcome.errors.push({ + failed++; + errors.push({ messageId: message.messageId, error: `invalid message timestamp: ${message.timestamp}`, }); @@ -241,24 +376,53 @@ async function writeSession( planned.push({ message, memoryId, - payload: { - id: memoryId, - content, - meta, - tree, - temporal, - }, + payload: { id: memoryId, content, meta, tree, temporal }, }); } - // A session JSONL can contain two events that share the same `event.uuid` - // (resume artefacts, sidechain merges, replay wrappers). The deterministic - // UUIDv7 is keyed on (tool, sessionId, messageId), so duplicates collapse - // to the same id and would otherwise hit the unique constraint server-side - // — taking the whole batch's transaction down with them. Drop them here. const dedup = dedupByMemoryId(planned); - outcome.skipped += dedup.duplicates; - const deduped = dedup.unique; + return { + planned: dedup.unique, + skipped: skipped + dedup.duplicates, + failed, + errors, + }; +} + +async function writeSession( + engine: MemoryClient, + session: ImportedSession, + title: string, + tree: string, + projectSlug: string, + gitRoot: string | undefined, + gitRemote: string | undefined, + options: WriteOptions, +): Promise { + const outcome: SessionOutcome = { + sessionId: session.sessionId, + title, + tree, + sourceFile: session.sourceFile, + inserted: 0, + updated: 0, + skipped: 0, + failed: 0, + errors: [], + }; + + const plan = planSession( + session, + tree, + projectSlug, + gitRoot, + gitRemote, + options, + ); + outcome.skipped += plan.skipped; + outcome.failed += plan.failed; + outcome.errors.push(...plan.errors); + const deduped = plan.planned; if (deduped.length === 0) return outcome; diff --git a/packages/cli/importers/slug.test.ts b/packages/cli/importers/slug.test.ts index 6f5c0a7..9f9735b 100644 --- a/packages/cli/importers/slug.test.ts +++ b/packages/cli/importers/slug.test.ts @@ -6,12 +6,7 @@ * `undefined` and the fallback to `basename(cwd)` is exercised. */ import { describe, expect, test } from "bun:test"; -import { - normalizeSlug, - repoNameFromRemote, - resolveProjectSlug, - SlugRegistry, -} from "./slug.ts"; +import { normalizeSlug, repoNameFromRemote, SlugRegistry } from "./slug.ts"; describe("repoNameFromRemote", () => { test("extracts the repo name from https and ssh remotes (sans .git)", () => { @@ -25,22 +20,6 @@ describe("repoNameFromRemote", () => { }); }); -describe("resolveProjectSlug", () => { - test("returns 'unknown' (no remote) for an empty/missing cwd", async () => { - expect(await resolveProjectSlug(undefined)).toEqual({ slug: "unknown" }); - expect(await resolveProjectSlug("")).toEqual({ slug: "unknown" }); - }); - - test("falls back to a normalized cwd basename when not in a git repo", async () => { - // /tmp/nonexistent-... isn't a git repo → no remote/root → basename. - const { slug, gitRemote } = await resolveProjectSlug( - "/tmp/nonexistent-path-xyz/memory-engine", - ); - expect(slug).toBe("memory_engine"); - expect(gitRemote).toBeUndefined(); - }); -}); - describe("normalizeSlug", () => { test("lowercases and replaces non-alphanumeric with underscore", () => { expect(normalizeSlug("Memory-Engine")).toBe("memory_engine"); diff --git a/packages/cli/importers/slug.ts b/packages/cli/importers/slug.ts index 2df6f71..c3567c9 100644 --- a/packages/cli/importers/slug.ts +++ b/packages/cli/importers/slug.ts @@ -3,8 +3,9 @@ * * A "slug" is an ltree-safe label derived from the session's cwd: * - Prefer the git `origin` remote's repo name (stable across clone locations, - * and shared with the Claude Code capture hook via `resolveProjectSlug`, so - * live + imported sessions for the same repo share one project label). + * and shared with the Claude Code capture hook via the import path + * (`importTranscriptFile`), so live + imported sessions for the same repo + * share one project label). * - Else the git repo root directory name, else `basename(cwd)`. * - Normalize to `[a-z0-9_]+`. * @@ -152,20 +153,6 @@ async function deriveBaseSlug( return { baseSlug: normalizeSlug(rawName), gitRoot, gitRemote }; } -/** - * Single-shot project context for one cwd (no cross-run collision registry) — - * used by the Claude Code capture hook so live captures nest under the same - * project label the import tool uses, and can record the git remote. Returns the - * `unknown` slug (and no remote) for an empty/missing cwd. - */ -export async function resolveProjectSlug( - cwd?: string, -): Promise<{ slug: string; gitRemote?: string }> { - if (!cwd || cwd.trim().length === 0) return { slug: UNKNOWN_SLUG }; - const { baseSlug, gitRemote } = await deriveBaseSlug(cwd); - return { slug: baseSlug, gitRemote }; -} - /** * Registry used to track slug assignments across a single import run so * colliding base slugs from different projects get distinct suffixes. From 4e9a8e47162df9c7d86ef3d852c9b3e5fdf1a03c Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 13:12:04 +0200 Subject: [PATCH 115/156] docs(decisions): record the Claude Code capture-hook change Document the switch from per-event capture (UserPromptSubmit + Stop, bespoke meta) to Stop/SessionEnd transcript import via the importer (dd28e26): rationale, the baked-in trade-offs (lost prompt-on-submit durability, SessionEnd flush, content_mode default), and how to change it. Co-Authored-By: Claude Opus 4.8 (1M context) --- DECISIONS_FOR_REVIEW.md | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index 2d7d630..9da9fd1 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -245,3 +245,55 @@ cheap interim guard: a test asserting every `core.principal` `kind='u'` has a matching `auth.users` and vice versa. **Status:** decided (defer); revisit with user-deletion / standalone users. + +--- + +## Claude Code plugin captures via the import path (Stop/SessionEnd transcript), not per-event + +**Date:** 2026-06-09 · **Area:** claude-plugin / capture hook (`dd28e26`) + +The plugin previously registered **`UserPromptSubmit`** (store your prompt) and +**`Stop`** (store the assistant's *final* message via `last_assistant_message`), +producing two memories per turn with a **bespoke metadata schema**. It now +registers **`Stop`** (after each turn) and **`SessionEnd`** (final flush), and +each fire reads `transcript_path` and runs the session through +`importTranscriptFile` — the *same* parse + write as `me … import`. So live +captures and bulk imports are **identical by construction**: same tree +(`share.projects..agent_sessions`), deterministic ids, and `source_*` +metadata, one memory per message. + +**Why:** the two capture paths had drifted (different metadata vocab — even a +conflicting `type` value — and the hook only caught the prompt + final message, +missing intermediate messages / tool calls / reasoning). Reusing the importer +gives parity, completeness, idempotency (deterministic message ids), and one +code path. Incremental + stateless: each fire does one `limit 1` newest-first +search for the session's high-water message (relies on the `orderBy`-desc default +fixed in `e9a6eec`) and writes only the delta; it falls back to the full +reconcile for a new session / `importer_version` bump / lost anchor / write error. + +**Maintainer decisions baked in:** + +- **Lost prompt-on-submit durability.** Dropping `UserPromptSubmit` means a turn + that never reaches `Stop`/`SessionEnd` (interrupt-and-quit, kill, API error) + isn't captured. Narrow: `Stop` fires per turn and re-imports the + session-so-far, so only the *last in-flight* turn is at risk. Keeping a + lightweight `UserPromptSubmit` safety-net was rejected — it reintroduces dual + paths and double-captures the prompt (the live copy has no message id to dedupe + against the transcript copy). +- **`SessionEnd` in addition to `Stop`** (vs `Stop`-only): a cheap final flush; + no local state to clean up since the watermark is server-derived. +- **`content_mode` default `"default"`** (user + assistant text). `full_transcript` + (reasoning + tool calls/results) is an opt-in plugin userConfig — off by default + because it's much larger/noisier and may capture sensitive tool output. + +**How to change it:** the hook command is `me claude hook --event stop|session-end` +([packages/cli/commands/claude.ts]) → `importTranscriptFile` +([packages/cli/importers/index.ts]); registered events live in +[packages/claude-plugin/hooks/hooks.json]. To restore prompt-on-submit capture, +re-add a `UserPromptSubmit` hook (and a dedupe story). To make `full_transcript` +the default, flip the `content_mode` userConfig default in +[packages/claude-plugin/.claude-plugin/plugin.json] (and the hook's +`resolveHookConfigFromEnv`). + +**Status:** decided (per request); document the capture model when the docs are +refreshed. From 5efb833ca488cd8744664f4635a4525e1d76ffa1 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 13:15:12 +0200 Subject: [PATCH 116/156] test(importers): assert hook capture and `me import` are cross-idempotent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both paths funnel through the same parse → SlugRegistry slug → tree → deterministic id, so importing a session via one and then the other must insert nothing the second time. Add tests for both orders (hook→import, import→hook) against the shared mock store, guarding the shared-derivation assumption against future drift (e.g. a divergent tree default or id keying). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cli/importers/import-transcript.test.ts | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/packages/cli/importers/import-transcript.test.ts b/packages/cli/importers/import-transcript.test.ts index 02557f2..e0720f8 100644 --- a/packages/cli/importers/import-transcript.test.ts +++ b/packages/cli/importers/import-transcript.test.ts @@ -11,9 +11,14 @@ import type { MemoryClient } from "../client.ts"; import { type Importer, importTranscriptFile, + runImport, type WriteOptions, } from "./index.ts"; -import type { ConversationMessage, ImportedSession } from "./types.ts"; +import type { + ConversationMessage, + ImportedSession, + ImporterOptions, +} from "./types.ts"; const WRITE: WriteOptions = { treeRoot: "share.projects", @@ -67,6 +72,18 @@ function importerFor(session: ImportedSession | null): Importer { }; } +/** An importer whose discoverSessions yields one fixed session (the `me import` path). */ +function discoverImporter(session: ImportedSession): Importer { + return { + tool: "claude", + defaultSource: "", + discoverSessions: async function* () { + yield session; + }, + parseFile: async () => session, + }; +} + /** Build a session whose messages have strictly-increasing timestamps. */ function session(messageIds: string[]): ImportedSession { const messages: ConversationMessage[] = messageIds.map((id, i) => ({ @@ -140,4 +157,49 @@ describe("importTranscriptFile", () => { expect(out?.inserted).toBe(1); // just "d" expect(store.size).toBe(4); }); + + // The hook (importTranscriptFile) and `me import` (runImport) must be + // idempotent w.r.t. each other: both derive the same tree + deterministic ids + // from the same parse, so importing a session via one path and then the other + // inserts nothing the second time. Guards the shared-derivation assumption. + test("hook capture then `me import` over the same session is a no-op", async () => { + const { client, store } = mockEngine(); + const s = session(["a", "b", "c"]); + + await importTranscriptFile(client, importerFor(s), "/x.jsonl", WRITE); + expect(store.size).toBe(3); + + const res = await runImport( + client, + discoverImporter(s), + {} as ImporterOptions, + WRITE, + ); + expect(res.inserted).toBe(0); + expect(res.skipped).toBe(3); + expect(store.size).toBe(3); + }); + + test("`me import` then hook capture over the same session is a no-op", async () => { + const { client, store } = mockEngine(); + const s = session(["a", "b", "c"]); + + const res = await runImport( + client, + discoverImporter(s), + {} as ImporterOptions, + WRITE, + ); + expect(res.inserted).toBe(3); + expect(store.size).toBe(3); + + const out = await importTranscriptFile( + client, + importerFor(s), + "/x.jsonl", + WRITE, + ); + expect(out?.inserted).toBe(0); + expect(store.size).toBe(3); + }); }); From fd28d4f690b7266bfc8ec4be2b3a3c42af6803b6 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 13:21:12 +0200 Subject: [PATCH 117/156] test(e2e): full-stack cross-idempotency of the capture hook and `me import` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an e2e scenario that writes a real Claude transcript, captures it via the real `me claude hook --event stop` (binary → server → Postgres), then runs `me claude import` over the same file and asserts the row count is unchanged, and re-running the hook is also a no-op. Exercises the actual server-side dedup (search_memory watermark + create_memory unique constraint), complementing the unit-level cross-idempotency tests. Uses two user turns so the importer doesn't skip it as trivial; a stdin-piping `meStdin` helper feeds the hook event JSON. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cli.e2e.test.ts | 94 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index f47d8bb..f870df5 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -19,7 +19,7 @@ process.env.SPACE_SCHEMA_PREFIX = "metest_"; import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { authStore } from "@memory.build/auth"; @@ -176,6 +176,37 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( return { stdout, stderr, code }; } + // Like `me`, but pipes `input` to the process's stdin (for `me claude hook`, + // which reads the event JSON from stdin). + async function meStdin( + args: string[], + input: string, + extraEnv?: Record, + ): Promise<{ stdout: string; stderr: string; code: number }> { + const proc = Bun.spawn([process.execPath, CLI, ...args], { + env: cliEnv(extraEnv), + stdin: new TextEncoder().encode(input), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const code = await proc.exited; + return { stdout, stderr, code }; + } + + // Count memories under a tree in this run's space schema. + async function countUnder(treePrefix: string): Promise { + const [row] = await sql.unsafe( + `select count(*)::int as n from metest_${spaceSlug}.memory + where tree <@ $1::ltree`, + [treePrefix], + ); + return (row?.n as number) ?? 0; + } + // Parse the --json stdout of a `me` invocation, asserting success. async function meJson( args: string[], @@ -322,6 +353,67 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(res.total).toBeGreaterThan(0); }); + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { + // A minimal Claude Code session transcript on disk. The importer scans + // //*.jsonl; the hook reads the file directly. + const sessionId = `xact-${rand()}`; + const root = await mkdtemp(join(tmpdir(), "me-e2e-transcript-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + const transcript = join(projDir, `${sessionId}.jsonl`); + // Two user turns so the importer doesn't skip it as a trivial session + // (the hook captures regardless; this makes both paths process all four). + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-02-01T00:00:0${i}.000Z`, + sessionId, + cwd: "/work/idempotent-proj", + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + const lines = [ + mkMsg(0, "user", "first question"), + mkMsg(1, "assistant", "first answer"), + mkMsg(2, "user", "second question"), + mkMsg(3, "assistant", "second answer"), + ]; + await writeFile( + transcript, + lines.map((l) => JSON.stringify(l)).join("\n"), + ); + + // cwd "/work/idempotent-proj" → no git repo on disk → slug = basename. + const tree = "share.projects.idempotent_proj.agent_sessions"; + + // 1. Live capture via the real hook (reads transcript_path from stdin, + // auths with the session, writes via importTranscriptFile). + const hook = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ transcript_path: transcript, session_id: sessionId }), + ); + expect(hook.code, hook.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + // 2. `me claude import` over the SAME transcript → no new rows (same tree + + // deterministic ids ⇒ the importer dedupes against the hook's writes). + const imp = await me(["claude", "import", "--source", root]); + expect(imp.code, imp.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + // 3. Re-run the hook → still idempotent. + const hook2 = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ transcript_path: transcript, session_id: sessionId }), + ); + expect(hook2.code, hook2.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + await rm(root, { recursive: true, force: true }); + }); + test("10. failure modes: bad space and missing auth exit non-zero", async () => { const badSpace = await me(["search", "--fulltext", "fox"], { ME_SPACE: "doesnotexist1", From 97d64c28c79640afb7292595fa1f8f548eed4f64 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 13:44:41 +0200 Subject: [PATCH 118/156] feat(memory): require an explicit tree on create/batchCreate `memory.create` and `memory.batchCreate` now require a non-empty `tree` at the protocol layer, so callers must choose `share` vs `~` deliberately instead of silently defaulting to the shared root. The MCP `me_memory_create` tool and the `me memory create` CLI command enforce and document this (most memories under `share`, private ones under `~`). The file importers remain the one place that defaults a tree-less record: `me memory import` and the `me_memory_import` MCP tool fall back to `share`. SHARE_NAMESPACE moves to @memory.build/protocol as the single source of truth (the wire default); @memory.build/database re-exports it. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- bun.lock | 1 + docs/cli/me-memory.md | 4 ++-- docs/mcp/me_memory_create.md | 2 +- packages/cli/commands/memory-import.ts | 4 +++- packages/cli/commands/memory.ts | 18 ++++++++++++++++-- packages/cli/mcp/server.ts | 18 ++++++++++-------- packages/database/package.json | 1 + packages/database/space/path.ts | 5 ++++- packages/protocol/fields.ts | 10 ++++++++++ packages/protocol/memory.ts | 4 ++-- .../rpc/memory/memory.integration.test.ts | 15 ++++++++------- packages/server/rpc/memory/memory.ts | 5 ++--- 13 files changed, 61 insertions(+), 28 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 63055f7..58f3f6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ Read the relevant docs before starting work on a subsystem. - **Memory table** (per space): `content`, `meta` (JSONB), `tree` (ltree), `temporal` (tstzrange), `embedding` (halfvec(1536)). - **Search**: hybrid BM25 + semantic via Reciprocal Rank Fusion, computed in SQL functions. - **Access**: no RLS. `core.build_tree_access(principalId, spaceId)` produces a `_tree_access` jsonb (rows of `tree_path` + `access`) passed into the space SQL functions (`search_memory`, `get_memory`, …). Three additive levels: **1 = read, 2 = write, 3 = owner**; `owner@root` (the empty ltree path) owns the whole space, and an owner grant at any path delegates access-management within that subtree. Two axes: **structural** authority (`principal_space.admin` — roster mutations, groups, invitations) vs **data** authority (owner@path); an admin may also grant data and can self-grant `owner@root`. The auth gate is a non-empty `build_tree_access` (every member holds ≥1 grant). -- **Tree conventions**: two reserved roots — per-member `home.` (`~` is input sugar for it; a joining **user** is granted `owner@home`) and the shared `share`. A space **creator** gets `admin` + `owner@home` + `owner@share`, **not** `owner@root` — so it sees `share` and its own `~` but not other members' homes (as an admin it can self-grant `owner@root`). A bare `memory.create` (no `tree`) defaults to `share` (`SHARE_NAMESPACE`). +- **Tree conventions**: two reserved roots — per-member `home.` (`~` is input sugar for it; a joining **user** is granted `owner@home`) and the shared `share`. A space **creator** gets `admin` + `owner@home` + `owner@share`, **not** `owner@root` — so it sees `share` and its own `~` but not other members' homes (as an admin it can self-grant `owner@root`). `memory.create`/`batchCreate` **require** an explicit `tree` (callers choose `share` vs `~` deliberately); only the file importers (`me memory import`, the `me_memory_import` MCP tool) default a tree-less record to `share` (`SHARE_NAMESPACE`, canonically defined in `@memory.build/protocol` and re-exported by `@memory.build/database`). - **API**: JSON-RPC 2.0 over HTTP, two endpoints: - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `invite.*`). - `/api/v1/user/rpc` — session only (an api key never authenticates here; agents can't manage agents). `whoami`, `agent.*`, `apiKey.*`, `space.*`. diff --git a/bun.lock b/bun.lock index 348d72d..ac7afbf 100644 --- a/bun.lock +++ b/bun.lock @@ -64,6 +64,7 @@ "name": "@memory.build/database", "version": "0.2.5", "dependencies": { + "@memory.build/protocol": "workspace:*", "@pydantic/logfire-node": "^0.13.1", "postgres": "^3.4.9", }, diff --git a/docs/cli/me-memory.md b/docs/cli/me-memory.md index 26790b5..7544ebf 100644 --- a/docs/cli/me-memory.md +++ b/docs/cli/me-memory.md @@ -34,11 +34,11 @@ me memory create [content] [options] | Option | Description | |--------|-------------| | `--content ` | Memory content (alternative to positional argument). | -| `--tree ` | Tree path (e.g., `work.projects.me`). | +| `--tree ` | **Required.** Tree path where the memory is stored (e.g., `share.work.projects`). Use `share` for memories the rest of the space should see, or `~` (your private home, e.g. `~.notes`) for memories that must stay private to you. | | `--meta ` | Metadata as a JSON string. | | `--temporal ` | Temporal range as `start[,end]` (ISO 8601). | -Content can come from the positional argument, the `--content` flag, or piped via stdin. +Content can come from the positional argument, the `--content` flag, or piped via stdin. A `--tree` path is required. --- diff --git a/docs/mcp/me_memory_create.md b/docs/mcp/me_memory_create.md index 3fcd449..dab3e87 100644 --- a/docs/mcp/me_memory_create.md +++ b/docs/mcp/me_memory_create.md @@ -9,7 +9,7 @@ Store a new memory. | `id` | `string \| null` | no | UUIDv7 for idempotent creates. Omit or pass `null` to auto-generate. | | `content` | `string` | yes | The content of the memory. Must be non-empty. | | `meta` | `object \| null` | no | Key-value metadata pairs. Omit or pass `null` to skip. | -| `tree` | `string \| null` | no | Hierarchical path using dot-separated labels (e.g., `work.projects.me`). Omit or pass `null` to store at the root. | +| `tree` | `string` | yes | Hierarchical path where the memory is stored, using dot-separated labels (e.g., `share.work.projects`). Choose deliberately: most memories should go under `share` so the rest of the space can see them; use `~` (your private home, e.g. `~.notes`) only for memories that must stay private to you. | | `temporal` | `object \| null` | no | Time range for the memory. Omit or pass `null` to skip. | ### temporal diff --git a/packages/cli/commands/memory-import.ts b/packages/cli/commands/memory-import.ts index 8b52a4d..106cdc1 100644 --- a/packages/cli/commands/memory-import.ts +++ b/packages/cli/commands/memory-import.ts @@ -9,6 +9,7 @@ import { existsSync, statSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import * as clack from "@clack/prompts"; +import { SHARE_NAMESPACE } from "@memory.build/protocol"; import { Command } from "commander"; import { batchCreateChunked } from "../chunk.ts"; import { resolveCredentials } from "../credentials.ts"; @@ -270,9 +271,10 @@ export function createMemoryImportCommand(): Command { const createParams = allMemories.map(({ memory: mem }) => ({ content: mem.content, + // tree is required on the wire; records without one default to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), })); diff --git a/packages/cli/commands/memory.ts b/packages/cli/commands/memory.ts index a05a7e2..2c77c54 100644 --- a/packages/cli/commands/memory.ts +++ b/packages/cli/commands/memory.ts @@ -100,7 +100,10 @@ function createMemoryCreateCommand(): Command { .description("create a memory") .argument("[content]", "memory content") .option("--content ", "memory content (alternative to positional)") - .option("--tree ", "tree path") + .option( + "--tree ", + "tree path ('share' for shared, '~' for private home)", + ) .option("--meta ", "metadata as JSON") .option("--temporal ", "temporal range (start[,end])") .action(async (positionalContent: string | undefined, opts, cmd) => { @@ -131,11 +134,22 @@ function createMemoryCreateCommand(): Command { process.exit(1); } + if (!opts.tree) { + if (fmt === "text") { + clack.log.error( + "No tree path provided. Pass --tree ('share' for shared memories, '~' for your private home).", + ); + } else { + output({ error: "No tree path provided" }, fmt, () => {}); + } + process.exit(1); + } + const client = buildMemoryClient(creds); try { const params: Record = { content }; - if (opts.tree) params.tree = opts.tree; + params.tree = opts.tree; if (opts.meta) params.meta = parseMeta(opts.meta); if (opts.temporal) params.temporal = parseTemporal(opts.temporal); diff --git a/packages/cli/mcp/server.ts b/packages/cli/mcp/server.ts index 8be8e93..3135e4b 100644 --- a/packages/cli/mcp/server.ts +++ b/packages/cli/mcp/server.ts @@ -8,6 +8,7 @@ import { existsSync, mkdirSync, statSync, writeFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { join, resolve } from "node:path"; +import { SHARE_NAMESPACE } from "@memory.build/protocol"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { stringify as yamlStringify } from "yaml"; @@ -72,10 +73,8 @@ Docs: ${docUrl("me_memory_create")}`, .describe("Key-value metadata pairs"), tree: z .string() - .optional() - .nullable() .describe( - "Hierarchical path (e.g., share.work.projects; ~.* for your private home). Omit or null to store under the shared root (`share`).", + "Hierarchical path where the memory is stored (required — choose deliberately). Most memories should go under `share` (e.g. `share.work.projects`) so the rest of the space can see them. Use `~` — your private home (e.g. `~.notes`) — only for memories that must stay private to you.", ), temporal: z .object({ @@ -104,7 +103,7 @@ Docs: ${docUrl("me_memory_create")}`, id: args.id ?? undefined, content: args.content, meta: args.meta ?? undefined, - tree: args.tree ?? undefined, + tree: args.tree, temporal: args.temporal ? { start: args.temporal.start, @@ -584,9 +583,9 @@ Docs: ${docUrl("me_memory_import")}`, const format = (args.format as ImportFormat) ?? undefined; const allMemories: Array<{ content: string; + tree: string; id?: string; meta?: Record; - tree?: string; temporal?: { start: string; end?: string }; }> = []; @@ -627,9 +626,10 @@ Docs: ${docUrl("me_memory_import")}`, for (const mem of memories) { allMemories.push({ content: mem.content, + // tree is required on the wire; default bare records to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), }); } @@ -645,9 +645,10 @@ Docs: ${docUrl("me_memory_import")}`, for (const mem of memories) { allMemories.push({ content: mem.content, + // tree is required on the wire; default bare records to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), }); } @@ -657,9 +658,10 @@ Docs: ${docUrl("me_memory_import")}`, for (const mem of memories) { allMemories.push({ content: mem.content, + // tree is required on the wire; default bare records to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), }); } diff --git a/packages/database/package.json b/packages/database/package.json index 7953e99..97a304b 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "dependencies": { + "@memory.build/protocol": "workspace:*", "@pydantic/logfire-node": "^0.13.1", "postgres": "^3.4.9" } diff --git a/packages/database/space/path.ts b/packages/database/space/path.ts index a9a5eaa..9d0b162 100644 --- a/packages/database/space/path.ts +++ b/packages/database/space/path.ts @@ -34,8 +34,11 @@ export const HOME_NAMESPACE = "home"; * `share/x` normalizes like any other path. It exists as a named constant * because membership/invitations grant a configurable level (read/write/owner) * at this root; see core `redeem_space_invitations`. + * + * Canonically defined in `@memory.build/protocol` (the wire contract) and + * re-exported here so the database/server side keeps a single source of truth. */ -export const SHARE_NAMESPACE = "share"; +export { SHARE_NAMESPACE } from "@memory.build/protocol"; /** A legal ltree label (PostgreSQL 16+): letters, digits, underscore, hyphen. */ const LTREE_LABEL = /^[A-Za-z0-9_-]+$/; diff --git a/packages/protocol/fields.ts b/packages/protocol/fields.ts index 330c10d..1e93f4e 100644 --- a/packages/protocol/fields.ts +++ b/packages/protocol/fields.ts @@ -51,6 +51,16 @@ export const treePathSchema = z "must be a tree path (labels [A-Za-z0-9_-], '.' or '/' separated, optional leading '~')", ); +/** + * The reserved shared-tree root. This is the single source of truth for the + * `"share"` literal across the codebase (the database boundary re-exports it). + * It is the conventional default for memories that should be visible to the + * whole space — `memory.create`/`batchCreate` now require an explicit `tree`, + * so callers that previously relied on a server-side default (the file + * importers) default to this. + */ +export const SHARE_NAMESPACE = "share"; + /** * Tree filter schema (ltree, lquery, or ltxtquery). * More permissive than treePathSchema since it allows query operators. diff --git a/packages/protocol/memory.ts b/packages/protocol/memory.ts index 4347870..068f982 100644 --- a/packages/protocol/memory.ts +++ b/packages/protocol/memory.ts @@ -23,7 +23,7 @@ export const memoryCreateParams = z.object({ id: uuidv7Schema.optional().nullable(), content: z.string().min(1, "content is required"), meta: metaSchema.optional().nullable(), - tree: treePathSchema.optional().nullable(), + tree: treePathSchema.min(1, "tree path is required"), temporal: temporalSchema.optional().nullable(), }); @@ -39,7 +39,7 @@ export const memoryBatchCreateParams = z.object({ id: uuidv7Schema.optional().nullable(), content: z.string().min(1, "content is required"), meta: metaSchema.optional().nullable(), - tree: treePathSchema.optional().nullable(), + tree: treePathSchema.min(1, "tree path is required"), temporal: temporalSchema.optional().nullable(), }), ) diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index 5252c84..bfe54d3 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -170,13 +170,14 @@ test("~ home + lenient separators normalize on input, reverse-map on output", as ); }); -test("create without a tree defaults to the shared root (`share`)", async () => { - // the default creator owns `share` (no root grant in beforeEach), so a bare - // create lands there and succeeds. - const created = await call<{ tree: string }>("memory.create", { - content: "shared by default", - }); - expect(created.tree).toBe("share"); +test("create without a tree is a validation error (tree is required)", async () => { + // `tree` is required on the wire — callers must choose `share` vs `~` + // explicitly. The file importers default to `share`, but a bare RPC create + // does not. + await expectAppError( + call("memory.create", { content: "no tree given" }), + "VALIDATION_ERROR", + ); }); test("create → get round-trips content/tree/meta and createdBy is null", async () => { diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 97b3ee6..15ebcab 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -9,7 +9,6 @@ * by id, desc (default, newest first) or asc; ranked/hybrid search ignores it * (score-desc). */ -import { SHARE_NAMESPACE } from "@memory.build/database"; import { generateEmbedding } from "@memory.build/embedding"; import { ACCESS } from "@memory.build/engine/core"; import type { @@ -183,7 +182,7 @@ async function memoryCreate( id: params.id ?? undefined, content: params.content, meta: params.meta ?? undefined, - tree: inputTreePath(ctx, params.tree ?? SHARE_NAMESPACE), + tree: inputTreePath(ctx, params.tree), temporal: formatTemporal(params.temporal), }), ); @@ -212,7 +211,7 @@ async function memoryBatchCreate( id: m.id ?? undefined, content: m.content, meta: m.meta ?? undefined, - tree: inputTreePath(ctx, m.tree ?? SHARE_NAMESPACE), + tree: inputTreePath(ctx, m.tree), temporal: formatTemporal(m.temporal), }), ); From 3ac9c18c3e97a8bac4af3c9a5041698a777d8c3d Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 14:04:20 +0200 Subject: [PATCH 119/156] docs: update for the principals/spaces model Bring docs/ in line with the multiplayer-branch redesign (engine/org/role -> principals/spaces). Rewrite the access-control and TypeScript-client guides, replace the "Engines" concept with "Spaces", and switch the MCP and getting-started guides to the session-token + X-Me-Space model. Retire the org/role/owner/engine/user/grant/invitation CLI references and add me-space, me-group, me-access, and me-agent. Update the docs-site nav to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/access-control.md | 174 +++++++++++-------------- docs/cli/me-access.md | 81 ++++++++++++ docs/cli/me-agent.md | 103 +++++++++++++++ docs/cli/me-apikey.md | 76 ++++------- docs/cli/me-engine.md | 101 --------------- docs/cli/me-grant.md | 87 ------------- docs/cli/me-group.md | 134 +++++++++++++++++++ docs/cli/me-invitation.md | 84 ------------ docs/cli/me-login.md | 24 +++- docs/cli/me-logout.md | 4 +- docs/cli/me-mcp.md | 13 +- docs/cli/me-memory.md | 4 +- docs/cli/me-org.md | 138 -------------------- docs/cli/me-owner.md | 73 ----------- docs/cli/me-pack.md | 10 +- docs/cli/me-role.md | 131 ------------------- docs/cli/me-serve.md | 10 +- docs/cli/me-space.md | 135 +++++++++++++++++++ docs/cli/me-user.md | 101 --------------- docs/cli/me-whoami.md | 6 +- docs/concepts.md | 40 +++--- docs/formats.md | 2 +- docs/getting-started.md | 19 ++- docs/index.md | 2 +- docs/mcp-integration.md | 34 ++--- docs/mcp/me_memory_import.md | 6 +- docs/memory-packs.md | 2 +- docs/troubleshooting.md | 11 +- docs/typescript-client.md | 237 ++++++++++++++++++---------------- packages/docs-site/lib/nav.ts | 11 +- 30 files changed, 801 insertions(+), 1052 deletions(-) create mode 100644 docs/cli/me-access.md create mode 100644 docs/cli/me-agent.md delete mode 100644 docs/cli/me-engine.md delete mode 100644 docs/cli/me-grant.md create mode 100644 docs/cli/me-group.md delete mode 100644 docs/cli/me-invitation.md delete mode 100644 docs/cli/me-org.md delete mode 100644 docs/cli/me-owner.md delete mode 100644 docs/cli/me-role.md create mode 100644 docs/cli/me-space.md delete mode 100644 docs/cli/me-user.md diff --git a/docs/access-control.md b/docs/access-control.md index 93f40ee..3dbaa02 100644 --- a/docs/access-control.md +++ b/docs/access-control.md @@ -1,142 +1,116 @@ # Access Control -Memory Engine uses tree-grant RBAC (Role-Based Access Control) enforced at the database level with PostgreSQL Row-Level Security. +Memory Engine organizes knowledge into **spaces**. Access within a space is granted on **tree paths**, not by role. There is no Row-Level Security — the server computes a caller's effective access and passes it into every database call. -## Users +## Principals -A user is a principal within an engine. Users can: +A **principal** is anything that can be granted access. There are three kinds: -- Own memories -- Receive grants to access tree paths -- Authenticate via API keys -- Belong to roles +| Kind | What it is | +|------|------------| +| **user** (`u`) | A human, authenticated by a session token (OAuth via GitHub or Google). | +| **agent** (`a`) | A service account owned by a user, authenticated by an API key. | +| **group** (`g`) | A named bundle of users and agents. | -Create a user: +A **member** is the user/agent sense only — the things that can be put into a group or hold an API key. Group membership is transitive: a member of a group gains the group's space membership, its admin (if the group is an admin), and all of its tree-access grants. -```bash -me user create alice -``` +## Spaces -Users with the `--superuser` flag bypass all access checks. Users with `--createrole` can create other users and roles. +A **space** is an isolated collection of memories with its own roster, groups, and access grants. Each space has: -Users created with `--no-login` are roles -- they cannot authenticate directly but can be granted access that members inherit. +- An immutable 12-character **slug** — also the `X-Me-Space` header value and the `me_` database schema name. +- A renamable display **name** (`me space rename` changes only this). -## Roles +A user can belong to many spaces; each memory lives in exactly one space. There are no organization, engine, or shard concepts above a space. -Roles group users together. When a grant is given to a role, all members of that role inherit the access. +## Two axes of authority -```bash -# Create a role -me role create engineering +Access splits into two independent axes: -# Add members -me role add-member engineering alice -me role add-member engineering bob +- **Structural authority** — `me space invite`, the roster (`me agent add`, `me group ...`), and invitations. This is the space **admin** flag. Admin transfers transitively through an admin group. Agents are never admins. +- **Data authority** — who can read/write/own memories at a given tree path. This is a **tree-access grant**. -# Grant access to the role (all members inherit it) -me grant create engineering work.projects read create update -``` +A space must always keep at least one *effective* admin (a user who is a direct admin or a member of an admin group). The last-admin safeguard rejects any removal or demotion that would drop it (error code `LAST_ADMIN`). -Roles are implemented as users with `canLogin: false`. This means grants work the same way for users and roles. +## Tree-access grants -## Grants +A grant attaches an access **level** to a principal at a **tree path**. Levels are additive: -Grants control what actions a user (or role) can perform on a tree path. A grant specifies: +| Level | Name | Capabilities | +|-------|------|--------------| +| 1 | **read** | Search and retrieve memories at or below the path. | +| 2 | **write** | Read + create, update, move, and delete memories. | +| 3 | **owner** | Write + manage access (grant/revoke) within the subtree. | -- **user** -- who receives the access -- **path** -- which tree path (and all descendants) -- **actions** -- what they can do: `read`, `create`, `update`, `delete` +Grants are **hierarchical**: a grant at `share.work` also covers `share.work.projects`, `share.work.projects.api`, and so on. An `owner` grant at a path delegates access-management for that whole subtree; `owner@root` (the empty path) owns the entire space. ```bash -# Grant read and create access to a tree branch -me grant create alice work.projects read create - -# Grant full access -me grant create bob work read create update delete +# Grant read access to a subtree +me access grant alice@example.com share.work r -# Check access -me grant check alice work.projects.api read -``` +# Grant write access +me access grant bob@example.com share.work.backend w -Grants are hierarchical -- a grant on `work` covers `work.projects`, `work.projects.api`, etc. +# Grant ownership of a subtree (lets the grantee manage access below it) +me access grant team-leads share.work o -### Actions +# List grants in the active space (optionally scope to one principal or path) +me access list +me access list alice@example.com +me access list --path share.work -| Action | Description | -|--------|-------------| -| `read` | Search and retrieve memories | -| `create` | Create new memories | -| `update` | Update existing memories | -| `delete` | Delete memories | - -Grant management and ownership are controlled separately: grants with `--with-grant-option` let a grantee re-grant their access, and ownership (`me owner set`) gives a user full admin access to a tree path. Superuser bootstrap is handled via the `superuser` flag on the user row, not via a grant action. - -### Grant option - -When creating a grant with `--with-grant-option`, the grantee can re-grant that same access to others: - -```bash -me grant create alice work.projects read create --with-grant-option +# Remove a grant +me access rm-grant bob@example.com share.work.backend ``` -Alice can now grant `read` and `create` on `work.projects` to other users. +The level argument accepts `r` (read), `w` (write), or `o` (owner). -## Ownership +## Reserved tree roots -Each tree path can have at most one owner. The owner has implicit admin access to that path and all descendants. +Every space has two conventional roots: -```bash -# Set owner -me owner set work.projects.api alice - -# Check owner -me owner get work.projects.api - -# List all ownership records -me owner list -``` - -Ownership is distinct from grants: +- **`share`** — the shared root. Memories everyone in the space should see go here. This is where the file importers default a tree-less record, and where `me memory create` / `me_memory_create` callers usually place memories. +- **`home.`** — a per-member private root. The input shortcut **`~`** expands to your own home, so `~.notes` means `home..notes` and displays back as `~.notes`. -- **Grants** are explicit, cumulative, and can be given to multiple users. -- **Ownership** is unique per path and provides automatic admin access. +`.` is the canonical path separator (`/` is also accepted on input and normalized). Labels must match `[A-Za-z0-9_-]`. -## How it works +### Default grants -Access control is enforced by PostgreSQL Row-Level Security (RLS) policies on the `me.memory` table. When a user authenticates with an API key, the database session is configured with their identity. Every query automatically checks whether the user has the required grant for the memory's tree path. +- A space **creator** gets `admin` + `owner@home` + `owner@share` — **not** `owner@root`. So the creator sees `share` and their own `~`, but not other members' homes. Because they're an admin, they can self-grant `owner@root` if they need the whole space. +- A **user** who joins a space is granted `owner@home` (their own private root). An admin then grants whatever shared access is appropriate (often via `me space invite --share`). -This means access control cannot be bypassed by application bugs -- it's enforced by the database itself. +## How it's enforced -:::warning[The invisible wall] -When RLS denies access, you get **empty results, not errors**. A search returns fewer results silently. A `memory.get` returns "not found" even if the memory exists. This is by design (PostgreSQL RLS behavior), but it can be confusing when debugging. +There is no Row-Level Security. For each request, the server calls `build_tree_access(principalId, spaceId)`, which collapses the principal's own grants and any inherited via group membership into a single set of `(tree_path, access)` rows. That set is passed as an argument into the space's SQL functions (`search_memory`, `get_memory`, …), which filter to the paths the caller may see. -If you're seeing missing results: +The authorization gate to use a space at all is holding **at least one** grant — every member has one (`owner@home` at minimum). -1. Check the user's access with `me grant check read` -2. Verify the memory exists by checking as a superuser -3. Check that the user has grants covering the memory's tree path -4. Remember that grants are hierarchical -- a grant on `work` covers `work.projects.*` +:::warning[Quiet filtering] +Access filtering happens inside the query. If you lack `read` on a memory's tree path, a search simply returns fewer rows and `me memory get` reports "not found" — you get no error distinguishing "doesn't exist" from "not visible to you." If you're missing results you expect, check your grants with `me access list `. ::: ## Example: team setup ```bash -# Create users -me user create alice -me user create bob -me user create carol - -# Create a shared role -me role create team -me role add-member team alice -me role add-member team bob -me role add-member team carol - -# Grant the team read access to everything -me grant create team "" read - -# Grant write access to specific branches -me grant create alice work.frontend read create update -me grant create bob work.backend read create update -me grant create carol work.infra read create update delete +# Create and enter a space (you become admin + owner@home + owner@share) +me space create "Acme Engineering" + +# Invite teammates by email; --share sets their access to the shared root +me space invite alice@example.com --share write +me space invite bob@example.com --share read +me space invite lead@example.com --admin --share owner + +# Group people for shared grants +me group create backend +me group add backend alice@example.com +me group add backend bob@example.com + +# Grant the group write access to a subtree (members inherit it) +me access grant backend share.work.backend w + +# Add one of your agents to the space and give it write access to share +me agent add ci-bot +me access grant ci-bot share w ``` + +See [`me access`](cli/me-access.md), [`me space`](cli/me-space.md), [`me group`](cli/me-group.md), and [`me agent`](cli/me-agent.md) for full command references. diff --git a/docs/cli/me-access.md b/docs/cli/me-access.md new file mode 100644 index 0000000..e8794cc --- /dev/null +++ b/docs/cli/me-access.md @@ -0,0 +1,81 @@ +# me access + +Manage tree-access grants in the active space. + +A grant attaches an access **level** to a principal (user, agent, or group) at a **tree path**. Levels are additive and hierarchical — a grant at `share.work` also covers everything below it: + +| Level | Flag | Capabilities | +|-------|------|--------------| +| read | `r` | Search and retrieve memories at or below the path. | +| write | `w` | Read + create, update, move, and delete memories. | +| owner | `o` | Write + manage access (grant/revoke) within the subtree. | + +`owner` at the empty (root) path owns the whole space. Granting access requires `owner` on the path in question (an admin can self-grant `owner@root`). See [Access Control](../access-control.md). + +These commands authenticate with your **session** and operate on the active space. + +## Commands + +- [me access grant](#me-access-grant) -- grant or update access at a path +- [me access rm-grant](#me-access-rm-grant) -- remove a grant +- [me access list](#me-access-list) -- list grants + +--- + +## me access grant + +Grant or update a principal's access at a tree path. + +``` +me access grant +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `principal` | yes | Principal id or name (user email, agent name, or group name). | +| `path` | yes | Tree path; use an empty string `""` for the space root. | +| `level` | yes | Access level: `r` (read), `w` (write), or `o` (owner). | + +```bash +me access grant alice@example.com share.work r +me access grant backend share.work.api w +me access grant lead@example.com "" o # owner@root — whole space +``` + +--- + +## me access rm-grant + +Remove a principal's grant at a tree path. + +``` +me access rm-grant +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `principal` | yes | Principal id or name. | +| `path` | yes | Tree path of the grant to remove. | + +--- + +## me access list + +List grants in the active space, optionally scoped to one principal and/or a path subtree. Alias: `me access ls`. + +``` +me access list [principal] [--path ] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `principal` | no | Filter to a single principal (id or name). | + +| Option | Description | +|--------|-------------| +| `--path ` | Only grants at or below this tree path. | + +## See also + +- [`me group`](me-group.md) -- grant to a group so all members inherit access. +- [`me space invite`](me-space.md#me-space-invite) -- set a new member's shared-root access at invite time. diff --git a/docs/cli/me-agent.md b/docs/cli/me-agent.md new file mode 100644 index 0000000..5dfc029 --- /dev/null +++ b/docs/cli/me-agent.md @@ -0,0 +1,103 @@ +# me agent + +Manage agents. + +An **agent** is a service account you own — a non-human principal that authenticates with an API key. Agents are **global** (owned by you, names unique per user), independent of any space. Create an agent, add it to the spaces it should work in, then mint it an API key with [`me apikey`](me-apikey.md). + +These commands authenticate with your **session** (`me login`). Lifecycle commands (`create`/`list`/`rename`/`delete`) are global; `add` and `groups` operate on the active space. + +## Commands + +- [me agent list](#me-agent-list) -- list your agents +- [me agent create](#me-agent-create) -- create an agent +- [me agent rename](#me-agent-rename) -- rename an agent +- [me agent delete](#me-agent-delete) -- delete an agent +- [me agent add](#me-agent-add) -- add an agent to the active space +- [me agent groups](#me-agent-groups) -- list an agent's groups in the space + +--- + +## me agent list + +List your agents. Alias: `me agent ls`. + +``` +me agent list +``` + +--- + +## me agent create + +Create an agent (a global service account you own). + +``` +me agent create +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `name` | yes | Agent name (unique among your agents). | + +--- + +## me agent rename + +Rename an agent. + +``` +me agent rename +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | +| `new-name` | yes | New name. | + +--- + +## me agent delete + +Delete an agent. Its API keys are deleted with it. Alias: `me agent rm`. + +``` +me agent delete +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | + +--- + +## me agent add + +Add one of your agents to the active space's roster. It joins with `owner@home`; grant it shared access with [`me access`](me-access.md). + +``` +me agent add +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | + +--- + +## me agent groups + +List the groups an agent belongs to in the active space. + +``` +me agent groups +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | + +## See also + +- [`me apikey`](me-apikey.md) -- mint, list, and revoke an agent's API keys. +- [`me access`](me-access.md) -- grant the agent access to tree paths. +- [MCP Integration](../mcp-integration.md) -- run an agent against a space over MCP. diff --git a/docs/cli/me-apikey.md b/docs/cli/me-apikey.md index b49f3a3..3e9fc7d 100644 --- a/docs/cli/me-apikey.md +++ b/docs/cli/me-apikey.md @@ -2,101 +2,76 @@ Manage API keys. -API keys authenticate users to an engine. Each key is scoped to a single user and can be used for MCP server connections, CLI authentication, and direct API access. +API keys are how **agents** authenticate. Each key belongs to one of your agents and is **global** — not bound to a space. The same key works in any space the agent has been admitted to; the space comes from the `X-Me-Space` header (`--space` / `ME_SPACE`). Keys are formatted `me..`. -## Commands - -- [me apikey list](#me-apikey-list) -- list API keys for a user -- [me apikey create](#me-apikey-create) -- create a new API key -- [me apikey show](#me-apikey-show) -- show a stored API key from credentials.yaml -- [me apikey revoke](#me-apikey-revoke) -- revoke an API key -- [me apikey delete](#me-apikey-delete) -- permanently delete an API key - ---- - -## me apikey list +Humans authenticate with a session (`me login`), not an API key. These commands authenticate with your **session**. -List API keys for a user. +The CLI never persists API keys. A created key is printed **once** for you to place where the agent runs (typically via the `ME_API_KEY` environment variable). The alias `me apikey revoke` is equivalent to `me apikey delete`. -``` -me apikey list -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | +## Commands -Displays a table of API keys with ID, name, last used date, and status. +- [me apikey create](#me-apikey-create) -- mint a key for an agent +- [me apikey list](#me-apikey-list) -- list an agent's keys +- [me apikey get](#me-apikey-get) -- show key metadata +- [me apikey delete](#me-apikey-delete) -- delete (revoke) a key --- ## me apikey create -Create a new API key. +Mint a new API key for one of your agents. The raw key is shown only once — store it securely. ``` -me apikey create [name] [options] +me apikey create [name] [--expires ] ``` | Argument | Required | Description | |----------|----------|-------------| -| `user` | yes | User name or ID. | +| `agent` | yes | Agent id or name. | | `name` | no | Key name (auto-generated if omitted). | | Option | Description | |--------|-------------| | `--expires ` | Expiration timestamp (ISO 8601). | -The raw key value is displayed only once at creation time. Store it securely. - --- -## me apikey show +## me apikey list -Show the API key stored locally in `credentials.yaml` for an engine. Reads only — no network call. +List an agent's API keys (metadata only — never the secret). Alias: `me apikey ls`. ``` -me apikey show [options] +me apikey list ``` -| Option | Description | -|--------|-------------| -| `--engine ` | Engine slug to look up. Defaults to the active engine for the resolved server. | - -The active server is resolved in the usual order (`--server` flag > `ME_SERVER` env > `default_server` in `credentials.yaml`). The active engine comes from that server's `active_engine` entry; switch it with `me engine use ` or override per-call with `--engine`. - -Errors when no engine can be resolved or when the named engine has no stored API key. - -Useful for scripting: +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | -```sh -export ME_API_KEY=$(me apikey show --json | jq -r .apiKey) -``` +Displays a table of keys with ID, name, last-used date, and expiry. --- -## me apikey revoke +## me apikey get -Revoke an API key. +Show metadata for a single API key. ``` -me apikey revoke +me apikey get ``` | Argument | Required | Description | |----------|----------|-------------| | `id` | yes | API key ID. | -Revokes the key (makes it inactive). The key record is retained but can no longer be used for authentication. - --- ## me apikey delete -Permanently delete an API key. +Permanently delete (revoke) an API key. There is no soft-revoke state — delete is the only way to invalidate a key. Irreversible. Aliases: `me apikey rm`, `me apikey revoke`. ``` -me apikey delete [options] +me apikey delete [-y] ``` | Argument | Required | Description | @@ -107,4 +82,7 @@ me apikey delete [options] |--------|-------------| | `-y, --yes` | Skip the confirmation prompt. | -This operation is irreversible. +## See also + +- [`me agent`](me-agent.md) -- create the agents that hold these keys and add them to spaces. +- [MCP Integration](../mcp-integration.md) -- supply a key to an MCP-connected agent via `--api-key` or `ME_API_KEY`. diff --git a/docs/cli/me-engine.md b/docs/cli/me-engine.md deleted file mode 100644 index e8db199..0000000 --- a/docs/cli/me-engine.md +++ /dev/null @@ -1,101 +0,0 @@ -# me engine - -Manage engines. - -An engine is an isolated memory database. Each engine has its own memories, users, roles, grants, and API keys. - -## Commands - -- [me engine list](#me-engine-list) -- list engines across all your organizations -- [me engine use](#me-engine-use) -- select the active engine -- [me engine create](#me-engine-create) -- create a new engine -- [me engine rename](#me-engine-rename) -- rename an engine -- [me engine delete](#me-engine-delete) -- permanently delete an engine - ---- - -## me engine list - -List engines across all your organizations. - -``` -me engine list -``` - -Displays a table of all engines you have access to, showing ID, name, slug, organization, and status. The active engine is marked. - ---- - -## me engine use - -Select the active engine. - -``` -me engine use [id-or-name] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | no | Engine ID or name. If omitted, an interactive picker is shown. | - -Switches the active engine. If no API key exists for the engine, one is created automatically. The active engine is used by all subsequent commands that interact with memories. - ---- - -## me engine create - -Create a new engine in an organization. - -``` -me engine create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | Engine name. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization ID. If omitted, an interactive picker is shown. | -| `--language ` | Text search language (default: `english`). | - ---- - -## me engine rename - -Rename an engine. - -``` -me engine rename -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | Engine ID or name. | -| `new-name` | yes | New engine name. | - -Renaming changes only the human-readable name. The engine slug -- which backs the underlying database schema (`me_`) and any stored API keys -- is randomly generated at creation time and never changes. Renaming does not invalidate the active engine selection or any API keys. - -Engine names must be unique within an organization. Renaming to a name already used by another engine in the same org fails with a `CONFLICT` error. - -Requires the `owner` or `admin` role on the organization that owns the engine. - ---- - -## me engine delete - -Permanently delete an engine and all its data. - -``` -me engine delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | Engine ID or name. | - -| Option | Description | -|--------|-------------| -| `--force` | Skip the confirmation prompt. | - -You will be asked to type the engine name to confirm unless `--force` is used. This operation is irreversible. diff --git a/docs/cli/me-grant.md b/docs/cli/me-grant.md deleted file mode 100644 index 5005b1e..0000000 --- a/docs/cli/me-grant.md +++ /dev/null @@ -1,87 +0,0 @@ -# me grant - -Manage tree grants. - -Grants control access to memories by tree path. A grant gives a user specific actions (read, create, update, delete) on a tree path and all its descendants. - -## Commands - -- [me grant create](#me-grant-create) -- grant tree access to a user -- [me grant revoke](#me-grant-revoke) -- revoke tree access -- [me grant list](#me-grant-list) -- list grants -- [me grant check](#me-grant-check) -- check if a user has access - ---- - -## me grant create - -Grant tree access to a user. - -``` -me grant create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | -| `path` | yes | Tree path to grant access to. | -| `actions...` | yes | One or more actions: `read`, `create`, `update`, `delete`. | - -| Option | Description | -|--------|-------------| -| `--with-grant-option` | Allow the grantee to re-grant this access to others. | - -### Example - -```bash -me grant create alice work.projects read create update -``` - ---- - -## me grant revoke - -Revoke tree access from a user. - -``` -me grant revoke -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | -| `path` | yes | Tree path to revoke access from. | - ---- - -## me grant list - -List grants. - -``` -me grant list [user] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | no | Filter by user name or ID. | - -Displays a table of grants with user, tree path, actions, and grant option. - ---- - -## me grant check - -Check if a user has access to a tree path. - -``` -me grant check -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | -| `path` | yes | Tree path. | -| `action` | yes | Action to check: `read`, `create`, `update`, `delete`. | - -Reports whether access is allowed or denied. diff --git a/docs/cli/me-group.md b/docs/cli/me-group.md new file mode 100644 index 0000000..fb23313 --- /dev/null +++ b/docs/cli/me-group.md @@ -0,0 +1,134 @@ +# me group + +Manage groups in the active space. + +A **group** is a named bundle of members (users and agents). Membership is **transitive**: a group member inherits the group's space membership, its admin flag (if the group is an admin), and all of its tree-access grants. Grant access to a group once and every member gets it. + +These commands authenticate with your **session** and operate on the active space. + +## Commands + +- [me group list](#me-group-list) -- list groups in the space +- [me group mine](#me-group-mine) -- list the groups you're in +- [me group create](#me-group-create) -- create a group +- [me group rename](#me-group-rename) -- rename a group +- [me group delete](#me-group-delete) -- delete a group +- [me group add](#me-group-add) -- add a member +- [me group remove](#me-group-remove) -- remove a member +- [me group members](#me-group-members) -- list a group's members + +--- + +## me group list + +List groups in the active space. Alias: `me group ls`. + +``` +me group list +``` + +--- + +## me group mine + +List the groups you are a member of in the active space. + +``` +me group mine +``` + +--- + +## me group create + +Create a group. + +``` +me group create +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `name` | yes | Group name. | + +--- + +## me group rename + +Rename a group. + +``` +me group rename +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | +| `new-name` | yes | New name. | + +--- + +## me group delete + +Delete a group. Alias: `me group rm`. + +``` +me group delete +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | + +--- + +## me group add + +Add a member (user or agent) to a group. + +``` +me group add [--admin] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | +| `member` | yes | User or agent id or name. | + +| Option | Description | +|--------|-------------| +| `--admin` | Make them a group admin (can manage the group's membership). | + +--- + +## me group remove + +Remove a member from a group. Alias: `me group rm-member`. + +``` +me group remove +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | +| `member` | yes | User or agent id or name. | + +--- + +## me group members + +List a group's members. + +``` +me group members +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | + +## See also + +- [`me access`](me-access.md) -- grant a group access to a tree path. +- [Access Control](../access-control.md) -- transitive membership and the authority model. diff --git a/docs/cli/me-invitation.md b/docs/cli/me-invitation.md deleted file mode 100644 index 4a3fc69..0000000 --- a/docs/cli/me-invitation.md +++ /dev/null @@ -1,84 +0,0 @@ -# me invitation - -Manage invitations. - -Invitations allow you to add people to an organization before they have an account. The invitee receives a token they can use to accept the invitation after signing up. - -## Commands - -- [me invitation create](#me-invitation-create) -- invite someone to an organization -- [me invitation list](#me-invitation-list) -- list pending invitations -- [me invitation accept](#me-invitation-accept) -- accept an invitation -- [me invitation revoke](#me-invitation-revoke) -- revoke a pending invitation - ---- - -## me invitation create - -Invite someone to an organization. - -``` -me invitation create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `email` | yes | Email address to invite. | -| `role` | yes | Role: `owner`, `admin`, or `member`. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID. | -| `--expires ` | Expiration in days (1-30, default: 7). | - -Displays the invitation ID, role, expiry, and the invitation token to share with the invitee. - ---- - -## me invitation list - -List pending invitations. - -``` -me invitation list [org] [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `org` | no | Organization name, slug, or ID. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID (alternative to positional argument). | - -Displays a table of pending invitations with ID, email, role, and expiry. - ---- - -## me invitation accept - -Accept an invitation. - -``` -me invitation accept -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `token` | yes | Invitation token received from the inviter. | - -You must be logged in to accept an invitation. - ---- - -## me invitation revoke - -Revoke a pending invitation. - -``` -me invitation revoke -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id` | yes | Invitation ID. | diff --git a/docs/cli/me-login.md b/docs/cli/me-login.md index dce715c..9592e13 100644 --- a/docs/cli/me-login.md +++ b/docs/cli/me-login.md @@ -5,14 +5,20 @@ Authenticate with Memory Engine via OAuth. ## Usage ``` -me login +me login [space] ``` +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | no | Space slug or name to make active after login. | + ## Description -Starts an OAuth device flow. You choose a provider (Google or GitHub), then the CLI displays a device code and opens your browser for authorization. Once you approve, the CLI stores your session token. +Starts an OAuth device flow. You choose a provider (Google or GitHub), the CLI displays a device code and opens your browser, and once you approve it stores your **session token**. Sessions are rolling: valid for 7 days and refreshed as you keep using the CLI. + +If you pass a `space` argument, it becomes the active space. Otherwise, if you belong to exactly one space it's selected automatically; if you belong to several, run `me space use` to pick one. The active space is carried as the `X-Me-Space` header on subsequent commands. -If your account has exactly one engine, it is automatically selected as the active engine and an API key is stored. +Login also runs the same version compatibility check as `me version` before opening the browser, so an out-of-date CLI gets a clean upgrade prompt instead of failing mid-flow. ## Global Options @@ -22,6 +28,12 @@ If your account has exactly one engine, it is automatically selected as the acti ## Notes -- Credentials are stored in `~/.config/me/credentials.yaml`. -- After login, use `me engine use` to select an engine if you have more than one. -- Use `me logout` to clear stored credentials. +- The session token is stored in your OS keychain when one is available (macOS `security`, Linux `secret-tool`); otherwise it falls back to `~/.config/me/credentials.yaml` (mode 0600). Set `ME_NO_KEYCHAIN=1` to force the file fallback. +- Non-secret settings (default server and per-server active space) live in `~/.config/me/config.yaml`. +- **API keys are for agents, not humans** — `me login` never creates one. Mint agent keys with [`me apikey create`](me-apikey.md#me-apikey-create). +- Use [`me logout`](me-logout.md) to clear the session; the non-secret config is kept so re-login resumes. + +## See also + +- [`me space`](me-space.md) -- list and switch the active space. +- [`me whoami`](me-whoami.md) -- show your identity and active space. diff --git a/docs/cli/me-logout.md b/docs/cli/me-logout.md index 44e7fb2..9dbb516 100644 --- a/docs/cli/me-logout.md +++ b/docs/cli/me-logout.md @@ -10,7 +10,9 @@ me logout ## Description -Removes all stored credentials (session token, API keys) for the active server from the credentials file. +Clears the stored **session token** for the active server — from the OS keychain when one is in use, and from `~/.config/me/credentials.yaml` otherwise. The non-secret config (default server and active space in `~/.config/me/config.yaml`) is kept, so a later `me login` resumes where you left off. + +Agent API keys are never persisted by the CLI (they only ever come from `ME_API_KEY`), so there is nothing to clear for agents. ## Global Options diff --git a/docs/cli/me-mcp.md b/docs/cli/me-mcp.md index 29383de..8753189 100644 --- a/docs/cli/me-mcp.md +++ b/docs/cli/me-mcp.md @@ -20,9 +20,16 @@ me mcp [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key for engine authentication. Can also be set via `ME_API_KEY` env var. | +| `--api-key ` | Agent API key. If omitted, the server uses your stored `me login` session. | +| `--space ` | Space to operate in (the `X-Me-Space`). | -The server URL is resolved from `--server` (global option) > `ME_SERVER` env > `https://api.memory.build`. +Resolution order: + +- **Auth token**: `--api-key` > `ME_API_KEY` > stored session token. +- **Space**: `--space` > `ME_SPACE` > stored active space. +- **Server URL**: `--server` (global option) > `ME_SERVER` > `https://api.memory.build`. + +A logged-in developer needs no key or space — the active session and active space are used automatically. For an unattended/headless agent, pass `--api-key` and `--space` (or set `ME_API_KEY` / `ME_SPACE`). This command is typically not run directly -- it is invoked by AI tools based on their MCP configuration. @@ -46,4 +53,4 @@ claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] ``` -Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `space` (and optionally `api_key`, `server`, `tree_root`). +Then start Claude Code, run `/plugin`, select `memory-engine`, and configure the options (all optional except `server`): leave `api_key` blank to use your `me login` session, leave `space` blank to use your active space, and `tree_root` defaults to `share.projects`. diff --git a/docs/cli/me-memory.md b/docs/cli/me-memory.md index 7544ebf..88100bd 100644 --- a/docs/cli/me-memory.md +++ b/docs/cli/me-memory.md @@ -235,11 +235,11 @@ Supports Markdown (with YAML frontmatter), YAML, JSON, and NDJSON. Format is aut ### Skipped memories -Memories with an explicit `id` that already exists in the engine are silently skipped server-side (via `ON CONFLICT DO NOTHING`) rather than failing the whole batch. The command surfaces these as `skipped` so re-imports of unchanged data and id collisions with unrelated memories are observable. Memories without an `id` get a server-generated UUIDv7 and never collide. +Memories with an explicit `id` that already exists in the space are silently skipped server-side (via `ON CONFLICT DO NOTHING`) rather than failing the whole batch. The command surfaces these as `skipped` so re-imports of unchanged data and id collisions with unrelated memories are observable. Memories without an `id` get a server-generated UUIDv7 and never collide. JSON output adds `skipped` (count) and `skippedIds` (array of conflicting ids). Text output appends `(K skipped — id already exists)` to the summary, or prints `Imported 0 memories (N already exist, no changes)` when everything was a re-import. Run with `--verbose` to see each skipped id inline. -Skipped memories do not contribute to the exit code; only parse and engine errors do. +Skipped memories do not contribute to the exit code; only parse and server errors do. `--dry-run` validates parsing only; it does not predict id collisions with already-imported memories. Run with `--verbose` after a real import to see the skipped ids. diff --git a/docs/cli/me-org.md b/docs/cli/me-org.md deleted file mode 100644 index f5098aa..0000000 --- a/docs/cli/me-org.md +++ /dev/null @@ -1,138 +0,0 @@ -# me org - -Manage organizations. - -Organizations group engines and members. Each engine belongs to exactly one organization. - -## Commands - -- [me org list](#me-org-list) -- list your organizations -- [me org create](#me-org-create) -- create an organization -- [me org rename](#me-org-rename) -- rename an organization -- [me org delete](#me-org-delete) -- delete an organization -- [me org member list](#me-org-member-list) -- list organization members -- [me org member add](#me-org-member-add) -- add a member -- [me org member remove](#me-org-member-remove) -- remove a member - ---- - -## me org list - -List your organizations. - -``` -me org list -``` - -Displays a table of organizations you belong to, showing ID, name, and slug. - ---- - -## me org create - -Create an organization. - -``` -me org create -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | Organization name. | - ---- - -## me org rename - -Rename an organization. - -``` -me org rename -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name-or-id` | yes | Organization name, slug, or ID. | -| `new-name` | yes | New organization name. | - -Renaming changes only the human-readable name. The org slug is randomly generated at creation time and never changes, so any references to the org by slug or ID continue to work. - -Requires the `owner` or `admin` role on the organization. - ---- - -## me org delete - -Delete an organization. - -``` -me org delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name-or-id` | yes | Organization name, slug, or ID. | - -| Option | Description | -|--------|-------------| -| `-y, --yes` | Skip the confirmation prompt. | - -This operation is irreversible. - ---- - -## me org member list - -List organization members. - -``` -me org member list [org] [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `org` | no | Organization name, slug, or ID. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID (alternative to positional argument). | - -Displays a table of members with name, email, role, and join date. - ---- - -## me org member add - -Add a member to an organization. - -``` -me org member add [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `email-or-id` | yes | Email address or identity ID of the person to add. | -| `role` | yes | Role: `owner`, `admin`, or `member`. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID. | - ---- - -## me org member remove - -Remove a member from an organization. - -``` -me org member remove [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name-email-or-id` | yes | Member name, email, or identity ID. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID. | -| `-y, --yes` | Skip the confirmation prompt. | diff --git a/docs/cli/me-owner.md b/docs/cli/me-owner.md deleted file mode 100644 index 5df4f4f..0000000 --- a/docs/cli/me-owner.md +++ /dev/null @@ -1,73 +0,0 @@ -# me owner - -Manage tree ownership. - -Ownership gives a user implicit admin access to a tree path and all its descendants. Unlike grants, ownership is unique per path -- each path has at most one owner. - -## Commands - -- [me owner set](#me-owner-set) -- set tree path owner -- [me owner remove](#me-owner-remove) -- remove tree path owner -- [me owner get](#me-owner-get) -- get tree path owner -- [me owner list](#me-owner-list) -- list ownership records - ---- - -## me owner set - -Set tree path owner. - -``` -me owner set -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `path` | yes | Tree path. | -| `user` | yes | User name or ID. | - ---- - -## me owner remove - -Remove tree path owner. - -``` -me owner remove -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `path` | yes | Tree path. | - ---- - -## me owner get - -Get tree path owner. - -``` -me owner get -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `path` | yes | Tree path. | - -Displays the path, owner name, set-by, and creation date. - ---- - -## me owner list - -List ownership records. - -``` -me owner list [user] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | no | Filter by user name or ID. | - -Displays a table of ownership records with tree path and owner. diff --git a/docs/cli/me-pack.md b/docs/cli/me-pack.md index ca4a42f..3b6fe65 100644 --- a/docs/cli/me-pack.md +++ b/docs/cli/me-pack.md @@ -2,12 +2,12 @@ Manage memory packs. -Memory packs are YAML files containing pre-built collections of memories. They provide structured knowledge that can be installed into any engine -- things like framework documentation, best practices, or domain-specific reference material. +Memory packs are YAML files containing pre-built collections of memories. They provide structured knowledge that can be installed into any space -- things like framework documentation, best practices, or domain-specific reference material. ## Commands - [me pack validate](#me-pack-validate) -- validate a pack file -- [me pack install](#me-pack-install) -- install a pack into the active engine +- [me pack install](#me-pack-install) -- install a pack into the active space - [me pack list](#me-pack-list) -- list installed packs --- @@ -30,7 +30,7 @@ Parses the YAML file and runs pack-specific constraint validation. Reports wheth ## me pack install -Install a memory pack into the active engine. +Install a memory pack into the active space. ``` me pack install [options] @@ -48,7 +48,7 @@ me pack install [options] The install process: 1. Validates the pack file. -2. Connects to the active engine. +2. Connects to the active space. 3. Finds existing memories from the same pack (by metadata). 4. Deletes stale memories from previous versions (with confirmation). 5. Creates all memories from the pack with `pack.*` tree prefixes and pack metadata. @@ -114,7 +114,7 @@ Dry-run output includes `wouldSkipIdempotent` (predicted from rows already at th ## me pack list -List installed packs in the active engine. +List installed packs in the active space. ``` me pack list diff --git a/docs/cli/me-role.md b/docs/cli/me-role.md deleted file mode 100644 index b173c16..0000000 --- a/docs/cli/me-role.md +++ /dev/null @@ -1,131 +0,0 @@ -# me role - -Manage roles. - -Roles are groups of users within an engine. Grant access to a role and all its members inherit that access. Roles cannot authenticate directly -- they are used purely for grouping. - -## Commands - -- [me role create](#me-role-create) -- create a role -- [me role delete](#me-role-delete) -- delete a role -- [me role list](#me-role-list) -- list all roles -- [me role add-member](#me-role-add-member) -- add a user to a role -- [me role remove-member](#me-role-remove-member) -- remove a user from a role -- [me role members](#me-role-members) -- list members of a role -- [me role list-for](#me-role-list-for) -- list roles a user belongs to - ---- - -## me role create - -Create a role. - -``` -me role create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | Role name. | - -| Option | Description | -|--------|-------------| -| `--identity-id ` | Link to an accounts identity. | - ---- - -## me role delete - -Delete a role. Alias: `me role rm`. - -``` -me role delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | Role ID or name. | - -| Option | Description | -|--------|-------------| -| `-y, --yes` | Skip confirmation prompt. | - -Deleting a role removes all grants and membership associations. This is irreversible. - ---- - -## me role list - -List all roles. - -``` -me role list -``` - -Displays a table of roles with ID and name. - ---- - -## me role add-member - -Add a user to a role. - -``` -me role add-member [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `role` | yes | Role ID or name. | -| `member` | yes | User ID or name. | - -| Option | Description | -|--------|-------------| -| `--with-admin-option` | Allow the member to manage this role. | - ---- - -## me role remove-member - -Remove a user from a role. - -``` -me role remove-member -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `role` | yes | Role ID or name. | -| `member` | yes | User ID or name. | - ---- - -## me role members - -List members of a role. - -``` -me role members -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `role` | yes | Role ID or name. | - -Displays a table of members with ID, name, and admin status. - ---- - -## me role list-for - -List roles a user belongs to. - -``` -me role list-for -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User ID or name. | - -Displays a table of roles with ID, name, and admin status. diff --git a/docs/cli/me-serve.md b/docs/cli/me-serve.md index a5bcd56..7f84c9a 100644 --- a/docs/cli/me-serve.md +++ b/docs/cli/me-serve.md @@ -13,13 +13,13 @@ me serve [--port ] [--host ] [--no-open] Starts a local HTTP server that: - Serves a React-based UI for browsing, searching, viewing, editing, and deleting memories. -- Proxies JSON-RPC calls from the browser to the configured engine, injecting your stored API key so the key never leaves the machine. +- Proxies JSON-RPC calls from the browser to the server, injecting your stored session token so it never leaves the machine. By default the server binds to `127.0.0.1:3000`; if 3000 is busy it tries 3001, 3002, … up to 3019 before giving up. Passing `--port` explicitly is strict — it does not auto-increment. The browser opens automatically on startup unless `--no-open` is passed. Press `Ctrl+C` to stop. -The UI talks to whichever engine is active for the current server — same resolution as every other `me` command (`--server` flag > `ME_SERVER` env > stored `default_server`; within the server, the active engine is picked via `me engine use`). Run `me whoami` to confirm. +The UI talks to whichever space is active for the current server — same resolution as every other `me` command (`--server` flag > `ME_SERVER` env > stored default server; within the server, the active space is picked via `me space use`). Run `me whoami` to confirm. ## Options @@ -60,7 +60,7 @@ The UI talks to whichever engine is active for the current server — same resol ## Security notes -- The server binds to `127.0.0.1` only — no LAN exposure. The browser never sees your API key or session token; `me serve` injects them into RPC calls on the way out. +- The server binds to `127.0.0.1` only — no LAN exposure. The browser never sees your session token; `me serve` injects it into RPC calls on the way out. - No authentication is required on the local server. Do not `--host 0.0.0.0` or tunnel the port unless you understand the implications. ## Examples @@ -72,11 +72,11 @@ me serve # Use a specific port and skip auto-open (handy when iterating in dev). me serve --port 8080 --no-open -# Point at a specific engine server. +# Point at a specific server. me serve --server https://api.memory.build ``` ## See also -- [`me engine use`](me-engine.md) — pick the active engine that `me serve` will connect to. +- [`me space use`](me-space.md#me-space-use) — pick the active space that `me serve` will connect to. - [`me memory search`](me-memory.md#me-memory-search) — the CLI equivalent of the UI's search bar. diff --git a/docs/cli/me-space.md b/docs/cli/me-space.md new file mode 100644 index 0000000..864c781 --- /dev/null +++ b/docs/cli/me-space.md @@ -0,0 +1,135 @@ +# me space + +Manage spaces. + +A **space** is an isolated collection of memories with its own roster, groups, and access grants. It is identified by an immutable 12-character **slug** (also the `X-Me-Space` header value and the `me_` database schema) and a renamable display **name**. Your *active* space is the one carried on every memory command; set it with `me space use` (or `me login `). + +These commands authenticate with your **session** (humans only — `me login`). Invitations operate on the active space. + +## Commands + +- [me space list](#me-space-list) -- list the spaces you belong to +- [me space use](#me-space-use) -- set the active space +- [me space create](#me-space-create) -- create a space +- [me space rename](#me-space-rename) -- rename a space +- [me space delete](#me-space-delete) -- delete a space +- [me space invite](#me-space-invite) -- invite a user (and manage invitations) + +--- + +## me space list + +List the spaces you belong to. The active space is marked. Alias: `me space ls`. + +``` +me space list +``` + +--- + +## me space use + +Set the active space (the `X-Me-Space` context used by other commands). Stored in `~/.config/me/config.yaml` per server. + +``` +me space use [space] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | no | Space slug or name. Prompts interactively if omitted. | + +--- + +## me space create + +Create a new space and make it active. As the creator you become a space **admin** and receive `owner@home` and `owner@share` (not `owner@root`). + +``` +me space create +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `name` | yes | Display name for the space. | + +--- + +## me space rename + +Rename a space's display name. The slug is immutable (it's the schema name and routing key), so this changes only the label. + +``` +me space rename +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | yes | Space slug or name. | +| `new-name` | yes | New display name. | + +--- + +## me space delete + +Permanently delete a space and all its data — memories, grants, groups, invitations. Irreversible. Alias: `me space rm`. + +``` +me space delete [--force] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | yes | Space slug or name. | + +| Option | Description | +|--------|-------------| +| `--force` | Skip the confirmation prompt. | + +--- + +## me space invite + +Invite a user to the active space by email. If the email already belongs to a registered user, they're added immediately; otherwise a pending invitation is recorded and redeemed at their next verified login. **Admin only.** + +``` +me space invite [--admin] [--share ] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `email` | yes | The invitee's email address. | + +| Option | Description | +|--------|-------------| +| `--admin` | Make the user a space admin (structural authority). | +| `--share ` | Access to grant at the shared root: `none`, `read`, `write`, or `owner` (default: `read`). | + +A joining user always receives `owner@home` (their private root); `--share` controls their access to `share`. + +### me space invite list + +List pending invitations for the active space. Alias: `me space invite ls`. + +``` +me space invite list +``` + +### me space invite revoke + +Revoke a pending invitation by email. + +``` +me space invite revoke +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `email` | yes | The invited email to revoke. | + +## See also + +- [Access Control](../access-control.md) -- principals, the two axes of authority, and tree-access grants. +- [`me access`](me-access.md) -- grant read/write/owner access on tree paths. +- [`me group`](me-group.md) -- bundle members for shared grants. +- [`me agent`](me-agent.md) -- add your agents to a space. diff --git a/docs/cli/me-user.md b/docs/cli/me-user.md deleted file mode 100644 index cd221cd..0000000 --- a/docs/cli/me-user.md +++ /dev/null @@ -1,101 +0,0 @@ -# me user - -Manage engine users. - -Users are principals within an engine that can own memories, receive grants, and authenticate via API keys. Each user belongs to a single engine. - -## Commands - -- [me user list](#me-user-list) -- list users -- [me user create](#me-user-create) -- create a user -- [me user get](#me-user-get) -- get a user by ID or name -- [me user delete](#me-user-delete) -- delete a user -- [me user rename](#me-user-rename) -- rename a user - ---- - -## me user list - -List users in the active engine. - -``` -me user list [options] -``` - -| Option | Description | -|--------|-------------| -| `--login-only` | Only show users that can log in (excludes roles). | - -Displays a table of users with ID, name, and flags (superuser, createrole, role). - ---- - -## me user create - -Create an engine user. - -``` -me user create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | User name. | - -| Option | Description | -|--------|-------------| -| `--superuser` | Grant superuser privileges. | -| `--createrole` | Allow this user to create other users and roles. | -| `--no-login` | Create as a role (cannot authenticate directly). | -| `--identity-id ` | Link to an accounts identity. | - ---- - -## me user get - -Get a user by ID or name. - -``` -me user get -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | User ID (UUIDv7) or name. | - -Displays full user details: name, ID, superuser, createrole, canLogin, identity, and creation date. - ---- - -## me user delete - -Delete a user. - -``` -me user delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | User ID or name. | - -| Option | Description | -|--------|-------------| -| `-y, --yes` | Skip the confirmation prompt. | - -This operation is irreversible. - ---- - -## me user rename - -Rename a user. - -``` -me user rename -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | User ID or current name. | -| `new-name` | yes | New name. | diff --git a/docs/cli/me-whoami.md b/docs/cli/me-whoami.md index 3381e86..de6425f 100644 --- a/docs/cli/me-whoami.md +++ b/docs/cli/me-whoami.md @@ -1,6 +1,6 @@ # me whoami -Show current identity and active engine. +Show current identity and active space. ## Usage @@ -10,9 +10,9 @@ me whoami ## Description -Fetches your current identity from the accounts API and displays your name, email, user ID, server URL, and active engine. +Calls the user endpoint (`whoami`) and displays your name, email, principal ID, server URL, and active space. -Returns an error if you are not logged in. +Returns an error if you are not logged in. Set or change the active space with [`me space use`](me-space.md#me-space-use). ## Global Options diff --git a/docs/concepts.md b/docs/concepts.md index 7271c10..31549b7 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -9,7 +9,7 @@ A memory is a single piece of knowledge. Every memory has: - **meta** -- key-value metadata for filtering (e.g., `{"type": "decision", "confidence": "high"}`). - **temporal** -- a time association, either a point-in-time or a date range. -Memories are stored in a single PostgreSQL table. There are no separate tables for different "types" of memory -- the type is a convention in `meta`, not a schema distinction. This keeps queries simple and the data model flexible. +Each space stores its memories in a single PostgreSQL table (the `me_` schema). There are no separate tables for different "types" of memory -- the type is a convention in `meta`, not a schema distinction. This keeps queries simple and the data model flexible. ### Best practices @@ -52,9 +52,18 @@ personal.reading personal.reading.books ``` -Tree paths use PostgreSQL's `ltree` extension. Labels must be **lowercase alphanumeric with underscores** (no spaces, hyphens, or uppercase). +Tree paths use PostgreSQL's `ltree` extension. Labels match `[A-Za-z0-9_-]` (letters, digits, underscores, and hyphens); use `.` as the separator (`/` is also accepted on input and normalized). -Keep paths **2-4 levels deep**. Deeper nesting rarely helps findability. When unsure about the right tree path, omit it -- you can always add one later, and content is still findable via search. +Keep paths **2-4 levels deep**. Deeper nesting rarely helps findability. + +### Reserved roots + +Every space has two conventional roots: + +- **`share`** -- the shared root. Memories the rest of the space should see go here (`share.work.projects`, etc.). The file importers default a tree-less record to `share`. +- **`home.`** -- your private per-member root. The input shortcut **`~`** expands to your own home, so `~.notes` is stored as `home..notes` and displays back as `~.notes`. + +`me memory create` (and the `me_memory_create` MCP tool) **require** an explicit tree -- choose `share` for shared memories or `~` for private ones. See [Access Control](access-control.md) for how grants attach to these paths. ### Tree filter syntax @@ -89,13 +98,13 @@ When filtering by tree (in search, export, or browse), the system auto-detects w ### Conventions -Tree paths are user-defined. There is no mandated hierarchy. Common patterns: +Below the two reserved roots, tree paths are user-defined. There is no mandated hierarchy. Common patterns: ``` -work.projects. # per-project knowledge -me.design. # design decisions -pack. # installed memory packs -notes. # general notes +share.work.projects. # shared per-project knowledge +share.design. # shared design decisions +pack. # installed memory packs (their own root) +~.notes. # private notes ``` ## Metadata @@ -211,13 +220,14 @@ Filters can also be used alone (without semantic or fulltext) to browse memories Search results include a `score` between 0 and 1, where 1 is the best match. For hybrid search, scores are computed via RRF fusion. For filter-only queries, results are sorted by creation time (configurable with `order_by`). -## Engines +## Spaces + +A **space** is an isolated collection of memories with its own roster, groups, and access grants. Each space has: -An engine is an isolated memory database. Each engine has its own: +- Its own memories (the `me_` table) and tree hierarchy. +- A roster of **principals** -- users, agents, and groups. +- Tree-access grants that control who can read/write/own which paths. -- Memories -- Users, roles, and grants -- API keys -- Tree hierarchy +A space is identified by an immutable 12-character **slug** (also the `X-Me-Space` header value) and a renamable display **name**. A user can belong to many spaces; each memory lives in exactly one space. There are no organization, engine, or shard concepts above a space. -Engines belong to organizations. A user can have access to multiple engines across multiple organizations, but each memory lives in exactly one engine. +Manage spaces with [`me space`](cli/me-space.md), and see [Access Control](access-control.md) for principals and grants. diff --git a/docs/formats.md b/docs/formats.md index e91c437..afa6454 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -11,7 +11,7 @@ Every memory has one required field (`content`) and four optional fields: | `id` | `string` | no | UUIDv7. Enables idempotent imports -- re-importing the same ID won't create a duplicate. Must match `^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`. | | `content` | `string` | **yes** | The memory text. Must be non-empty. | | `meta` | `object` | no | Arbitrary key-value metadata. Any valid JSON object. | -| `tree` | `string` | no | Hierarchical path using dot-separated labels (e.g. `work.projects.api`). Labels must be alphanumeric or underscore. Must match `^([A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*)?$`. | +| `tree` | `string` | no | Hierarchical path using dot-separated labels (e.g. `share.work.projects.api`). Labels match `[A-Za-z0-9_-]`; `/` is also accepted as a separator and a leading `~` expands to your private home. When omitted, the file importers (`me memory import`, `me_memory_import`) default the record to the shared root `share`. | | `temporal` | varies | no | Time range for the memory. Accepted shapes depend on format -- see below. | ### Temporal input shapes diff --git a/docs/getting-started.md b/docs/getting-started.md index e167156..9256fc8 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -16,7 +16,16 @@ This installs the `me` binary to `~/.local/bin`. Make sure it's on your PATH. me login ``` -This starts an OAuth flow via GitHub -- authorize in your browser and the CLI stores your session. +This starts an OAuth device flow via GitHub or Google -- authorize in your browser and the CLI stores your session token (rolling 7-day, refreshed as you use it). On a host with a system keychain the token is stored there; otherwise it falls back to `~/.config/me/credentials.yaml` (mode 0600). + +If you belong to more than one space, pick the active one (it's carried as the `X-Me-Space` on every request): + +```bash +me space list +me space use +``` + +`me login ` selects it in one step, and `me whoami` shows your identity and active space. If your CLI is older than the server (or vice versa), `me login` will tell you and bail out before sending you to the browser. You can run the same check explicitly: @@ -29,10 +38,12 @@ me version ```bash me memory create "PostgreSQL 18 supports native UUIDv7 generation." \ - --tree notes.postgres \ + --tree share.notes.postgres \ --meta '{"topic": "database"}' ``` +A `--tree` is required. Put memories the rest of your space should see under `share.*`, and personal ones under `~.*` (your private home). See [Core Concepts](concepts.md#reserved-roots). + ## Search ```bash @@ -79,7 +90,7 @@ claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine ``` -Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `space` (and optionally `api_key`, `server`, `tree_root`). +Then start Claude Code, run `/plugin`, select `memory-engine`, and configure the options. All are optional except `server`: leave `api_key` blank to use your `me login` session, leave `space` blank to use your active space, and `tree_root` defaults to `share.projects`. After installation, your AI agent has access to memory tools -- create, search, get, update, delete, and more. @@ -88,7 +99,7 @@ See [MCP Integration](mcp-integration.md) for details. ## What's next - [Core Concepts](concepts.md) -- understand memories, tree paths, metadata, search modes -- [Access Control](access-control.md) -- users, roles, grants, and ownership +- [Access Control](access-control.md) -- spaces, principals, and tree-access grants - [Memory Packs](memory-packs.md) -- install pre-built knowledge collections - [MCP Integration](mcp-integration.md) -- how AI agents use Memory Engine - [CLI Reference](cli/me-memory.md) -- full command reference diff --git a/docs/index.md b/docs/index.md index 5bb6cf4..92fb9ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ Permanent memory for AI agents. Store, search, and organize knowledge across con - [Getting Started](getting-started.md) -- install, login, first memory - [Core Concepts](concepts.md) -- memories, tree paths, metadata, search modes -- [Access Control](access-control.md) -- users, roles, grants, ownership +- [Access Control](access-control.md) -- spaces, principals, tree-access grants - [Memory Packs](memory-packs.md) -- pre-built knowledge collections - [MCP Integration](mcp-integration.md) -- connecting AI agents - [TypeScript Client](typescript-client.md) -- programmatic access from TypeScript/JavaScript diff --git a/docs/mcp-integration.md b/docs/mcp-integration.md index 0318ce0..8a134b0 100644 --- a/docs/mcp-integration.md +++ b/docs/mcp-integration.md @@ -12,17 +12,17 @@ When an AI tool launches `me mcp`, it spawns a child process that communicates o └──────────────┘ └──────────┘ └────────────────┘ ``` -Authentication is baked into the command via `--api-key` and `--server` flags. The AI agent never sees or handles credentials — it just calls MCP tools and gets results back. +The AI agent never sees or handles credentials — it just calls MCP tools and gets results back. -Each `me mcp` instance is locked to a single engine via its API key. The MCP server does **not** read the credentials file — the API key must be provided via `--api-key` or the `ME_API_KEY` environment variable. The server URL defaults to `https://api.memory.build` (the hosted engine) but can be overridden with `--server` or `ME_SERVER`. +Each `me mcp` instance is locked to a single **space**, carried as the `X-Me-Space` header. The space is resolved from `--space` > `ME_SPACE` > your stored active space. Authentication is **either** an agent API key (`--api-key` or `ME_API_KEY`) **or**, if no key is given, your stored `me login` session token — so a developer install needs no key at all. The server URL defaults to `https://api.memory.build` but can be overridden with `--server` or `ME_SERVER`. ## Setup ### Prerequisites -You need an API key. Run `me whoami` to see your active engine, or create an API key with `me apikey create`. +Log in with `me login` and select a space — `me whoami` shows your active space and identity. That session is enough to run the MCP server locally. For an unattended or dedicated-agent install, mint an API key with `me apikey create ` and pass it with `--api-key`. -The server defaults to `https://api.memory.build`. Pass `--server ` only if you're running a self-hosted engine. +The server defaults to `https://api.memory.build`. Pass `--server ` only if you're running a self-hosted server. ### Agent-specific installers @@ -32,7 +32,7 @@ me codex install me gemini install ``` -These commands register Memory Engine with the named tool. They read your API key and server URL from the credentials file and bake them into the tool's MCP configuration, so the `me mcp` process can authenticate without the credentials file. +These commands register Memory Engine with the named tool, writing a `me mcp` invocation into the tool's MCP configuration. By default they embed no key — the server uses your `me login` session at runtime. Pass `--api-key` to pin a dedicated agent key instead, `--space ` to pin a space, and `--server ` to pin a non-default server. See the agent-specific command references for details: [`me opencode install`](cli/me-opencode.md#me-opencode-install), [`me codex install`](cli/me-codex.md#me-codex-install), and [`me gemini install`](cli/me-gemini.md#me-gemini-install). @@ -61,7 +61,7 @@ me gemini install To configure manually: ```bash -gemini mcp add --scope user me me mcp --api-key --server +gemini mcp add --scope user me me mcp --api-key --space --server ``` ### Codex CLI @@ -73,7 +73,7 @@ me codex install To configure manually: ```bash -codex mcp add me -- me mcp --api-key --server +codex mcp add me -- me mcp --api-key --space --server ``` ### OpenCode @@ -85,7 +85,7 @@ codex mcp add me -- me mcp --api-key --server "mcp": { "me": { "type": "local", - "command": ["me", "mcp", "--api-key", "", "--server", ""] + "command": ["me", "mcp", "--api-key", "", "--space", "", "--server", ""] } } } @@ -100,7 +100,7 @@ Add a `.vscode/mcp.json` file to your workspace: "servers": { "me": { "command": "me", - "args": ["mcp", "--api-key", "", "--server", ""] + "args": ["mcp", "--api-key", "", "--space", "", "--server", ""] } } } @@ -119,7 +119,7 @@ Open your Zed settings (`Zed > Settings > Open Settings` or `~/.config/zed/setti "context_servers": { "me": { "command": "me", - "args": ["mcp", "--api-key", "", "--server", ""] + "args": ["mcp", "--api-key", "", "--space", "", "--server", ""] } } } @@ -132,7 +132,7 @@ After saving, check the Agent Panel settings — the indicator next to "me" shou Any tool that supports the MCP stdio transport can use Memory Engine. The server command is: ```bash -me mcp --api-key --server +me mcp --api-key --space --server ``` Point your client at this command with `stdio` as the transport type. @@ -176,9 +176,9 @@ This project uses Memory Engine for persistent knowledge. ## Memory Map -- `design.*` -- architecture decisions and design docs -- `research.*` -- research findings and comparisons -- `bugs.*` -- known issues and workarounds +- `share.design.*` -- architecture decisions and design docs +- `share.research.*` -- research findings and comparisons +- `share.bugs.*` -- known issues and workarounds ## How to Search @@ -199,7 +199,7 @@ me_memory_search({semantic: "how does authentication work"}) me_memory_search({fulltext: "OAuth JWT"}) # Browse a section -me_memory_search({tree: "design.*"}) +me_memory_search({tree: "share.design.*"}) ``` ## Troubleshooting @@ -207,11 +207,11 @@ me_memory_search({tree: "design.*"}) ### MCP server shows "failed" or "disabled" 1. Verify the `me` binary is on your PATH: `which me` -2. Test the server directly: `echo '{}' | me mcp --api-key --server ` +2. Test the server directly: `echo '{}' | me mcp --api-key --space --server ` 3. Re-install with the agent-specific command, for example `me opencode install`, `me codex install`, or `me gemini install`. For Claude Code, open `/plugin` and reconfigure `memory-engine`. ### Agent can't find memories -1. Check that the correct engine is active: `me whoami` +1. Check that the correct space is active: `me whoami` 2. Verify memories exist: `me memory search --fulltext ""` 3. Check that embeddings have been computed: `me memory get ` (look for `hasEmbedding: true`) diff --git a/docs/mcp/me_memory_import.md b/docs/mcp/me_memory_import.md index 89b4ce2..a9bef34 100644 --- a/docs/mcp/me_memory_import.md +++ b/docs/mcp/me_memory_import.md @@ -18,7 +18,7 @@ One of `path` or `content` must be provided. JSON (array or single object), NDJSON, YAML (array or single object), and Markdown (YAML frontmatter + body, one memory per file). -Each memory object supports fields: `id`, `content` (required), `meta`, `tree`, `temporal`. +Each memory object supports fields: `id`, `content` (required), `meta`, `tree`, `temporal`. Unlike `me_memory_create` (which requires an explicit `tree`), a record with no `tree` is imported into the shared root `share`. See [File Formats](../formats.md) for full schema documentation, examples, and format detection rules. @@ -43,13 +43,13 @@ See [File Formats](../formats.md) for full schema documentation, examples, and f | Field | Type | Description | |-------|------|-------------| | `imported` | `number` | Number of memories successfully imported on this call. | -| `skipped` | `number` | Number of memories whose explicit `id` already existed in the engine. Always present (may be `0`). | +| `skipped` | `number` | Number of memories whose explicit `id` already existed in the space. Always present (may be `0`). | | `failed` | `number` | Number of memories in chunks that errored before reaching the server. Always present (may be `0`). | | `ids` | `string[]` | UUIDs of the memories actually inserted on this call. | | `skippedIds` | `string[]` | The explicit ids that were skipped because they already existed. Always present (may be empty). Inspect any of these with `me_memory_get` to see what's there. | | `errors` | `Array<{ chunkIndex, itemCount, ids, error }>` | One entry per failed chunk. Always present (may be empty). | -The tool is idempotent for memories with explicit ids: re-calling with the same arguments leaves the engine in the same state, with all previously-imported ids appearing in `skippedIds` instead of `ids`. Memories submitted without an explicit `id` get a server-generated UUIDv7 and never collide. +The tool is idempotent for memories with explicit ids: re-calling with the same arguments leaves the space in the same state, with all previously-imported ids appearing in `skippedIds` instead of `ids`. Memories submitted without an explicit `id` get a server-generated UUIDv7 and never collide. ### Chunking and partial failures diff --git a/docs/memory-packs.md b/docs/memory-packs.md index d83b723..e752828 100644 --- a/docs/memory-packs.md +++ b/docs/memory-packs.md @@ -8,7 +8,7 @@ Memory packs are YAML files containing pre-built collections of memories. They s # Validate first (offline, no server needed) me pack validate packs/typescript-best-practices.yaml -# Install into the active engine +# Install into the active space me pack install packs/typescript-best-practices.yaml ``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c6ed052..89ab163 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7,17 +7,17 @@ 1. **Check embedding status** -- semantic search requires embeddings. New memories take ~10-30 seconds to get embeddings. Use `me memory get ` and check `hasEmbedding`. 2. **Try fulltext instead** -- fulltext search works immediately after creation. Use `--fulltext` to search by keywords. 3. **Broaden the search** -- remove filters (tree, meta, temporal) to see if results appear without them. -4. **Check access** -- RLS silently filters results. If you're missing memories you know exist, check grants with `me grant check read`. See [Access Control](access-control.md) for details. -5. **Check as superuser** -- superusers bypass all access checks. If results appear for a superuser but not a regular user, the issue is grants. +4. **Check the active space** -- results come only from your active space. Run `me whoami` to confirm it, and `me space use ` to switch. +5. **Check access** -- the server filters results to the tree paths you can read; a missing grant looks like missing results, not an error. List your grants with `me access list ` (or `me access list --path `). See [Access Control](access-control.md) for details. ### "Memory not found" on get or update This can mean either: -- The memory genuinely doesn't exist (wrong ID) -- The memory exists but the current user doesn't have `read` access to its tree path (RLS returns "not found") +- The memory genuinely doesn't exist (wrong ID), or it's in a different space than your active one +- The memory exists but you don't have `read` access to its tree path (access filtering reports it as "not found") -Check access with `me grant check read` or retry as a superuser. +Confirm your active space with `me whoami` and check your grants with `me access list `. ### Embeddings stuck @@ -50,5 +50,6 @@ These all use code `-32000` but are distinguished by `data.code`: | `FORBIDDEN` | Valid credentials, insufficient permissions | Check grants for the required action | | `NOT_FOUND` | Resource doesn't exist (or no access) | Verify the ID and check grants | | `CONFLICT` | Resource already exists (e.g., duplicate slug) | Use a different identifier | +| `LAST_ADMIN` | Operation would leave the space with no effective admin | Promote another user/admin group first | | `RATE_LIMITED` | Too many requests | Back off and retry after the `Retry-After` header | | `VALIDATION_ERROR` | Business logic validation failed | Check the error message for details | diff --git a/docs/typescript-client.md b/docs/typescript-client.md index 376cbf8..4933d7e 100644 --- a/docs/typescript-client.md +++ b/docs/typescript-client.md @@ -8,20 +8,30 @@ The `@memory.build/client` package provides programmatic access to Memory Engine npm install @memory.build/client ``` +## Two clients + +The package exposes two clients, matching the two API endpoints: + +- **`createMemoryClient`** — the space data plane plus space management. Talks to `POST /api/v1/memory/rpc`, carrying the active space in the `X-Me-Space` header. Authenticates with **either** a session token **or** an agent API key. Namespaces: `memory`, `principal`, `group`, `grant`, `invite`. +- **`createUserClient`** — session-only, user-scoped operations. Talks to `POST /api/v1/user/rpc`. Authenticates with a **session token only** (an API key can't manage agents). Methods: `whoami`, plus the `agent`, `apiKey`, and `space` namespaces. + +There is also `createAuthClient` for the OAuth device-flow login that produces a session token. + ## Quick start ```typescript -import { createClient } from "@memory.build/client"; +import { createMemoryClient } from "@memory.build/client"; -const me = createClient({ - url: "https://api.memory.build", - apiKey: "me.xxx.yyy", +const me = createMemoryClient({ + url: "https://api.memory.build", // default + token: sessionTokenOrApiKey, // session token or "me.." + space: "abc123def456", // the X-Me-Space slug }); -// Create a memory +// Create a memory (tree is required — choose share.* or ~.* deliberately) await me.memory.create({ content: "TypeScript was released in 2012", - tree: "knowledge.programming", + tree: "share.knowledge.programming", }); // Search @@ -33,24 +43,32 @@ const { results } = await me.memory.search({ ## Configuration ```typescript -const me = createClient({ +const me = createMemoryClient({ url: "https://api.memory.build", // default - apiKey: "me.xxx.yyy", // format: "me.." - timeout: 30000, // request timeout in ms (default: 30000) - retries: 3, // automatic retries (default: 3) + token: "me.xxx.yyy", // session token or api key (format: "me..") + space: "abc123def456", // active space slug (sent as X-Me-Space) + timeout: 30000, // request timeout in ms (default: 30000) + retries: 3, // automatic retries (default: 3) }); ``` -The client retries on `429`, `500`, `502`, `503`, and `504` responses with exponential backoff and jitter. It respects the `Retry-After` header. +The client retries on `429`, `500`, `502`, `503`, and `504` responses with exponential backoff and jitter, and respects the `Retry-After` header. The token and space can be swapped at runtime: + +```typescript +me.setToken("me.newkey.newsecret"); +me.setSpace("otherslug1234"); +``` ## Memory operations ### create +`tree` is required. Use `share.*` for memories the rest of the space should see, or `~.*` for your private home. + ```typescript const memory = await me.memory.create({ content: "The fact to remember", - tree: "work.projects.acme", // optional hierarchical path + tree: "share.work.projects.acme", // required meta: { source: "meeting-notes" }, // optional JSON metadata temporal: { // optional time range start: "2025-01-01T00:00:00Z", @@ -62,13 +80,13 @@ const memory = await me.memory.create({ ### batchCreate -Create up to 1,000 memories in a single call. +Create up to 1,000 memories in a single call. Each memory requires a `tree`. ```typescript const { ids } = await me.memory.batchCreate({ memories: [ - { content: "First memory", tree: "notes" }, - { content: "Second memory", tree: "notes" }, + { content: "First memory", tree: "share.notes" }, + { content: "Second memory", tree: "share.notes" }, ], }); ``` @@ -102,11 +120,8 @@ const { deleted } = await me.memory.delete({ id: "019..." }); Delete all memories under a tree prefix. ```typescript -// Preview what would be deleted -const { count } = await me.memory.deleteTree({ tree: "old.project", dryRun: true }); - -// Actually delete -const { count: deleted } = await me.memory.deleteTree({ tree: "old.project" }); +const { count } = await me.memory.deleteTree({ tree: "share.old.project", dryRun: true }); +const { count: deleted } = await me.memory.deleteTree({ tree: "share.old.project" }); ``` ### move @@ -115,8 +130,8 @@ Move memories from one tree prefix to another, preserving subtree structure. ```typescript const { count } = await me.memory.move({ - source: "drafts.api", - destination: "published.api", + source: "share.drafts.api", + destination: "share.published.api", }); ``` @@ -126,10 +141,9 @@ View the hierarchical tree structure with counts at each node. ```typescript const { nodes } = await me.memory.tree(); -// [{ path: "work", count: 5 }, { path: "work.projects", count: 3 }, ...] +// [{ path: "share", count: 5 }, { path: "share.work", count: 3 }, ...] -// Scoped to a subtree -const { nodes } = await me.memory.tree({ tree: "work", levels: 2 }); +const { nodes } = await me.memory.tree({ tree: "share.work", levels: 2 }); ``` ## Search @@ -137,14 +151,14 @@ const { nodes } = await me.memory.tree({ tree: "work", levels: 2 }); The `search` method supports keyword, semantic, and hybrid search with multiple filter types. ```typescript -const { results, total } = await me.memory.search({ +const { results } = await me.memory.search({ // Search modes (use one or both for hybrid) semantic: "natural language meaning query", fulltext: "exact keyword BM25 match", // Filters (all optional, combined with AND) grep: "regex.*pattern", // POSIX regex on content - tree: "work.projects.*", // ltree/lquery filter + tree: "share.work.projects.*", // ltree/lquery filter meta: { source: "meeting-notes" }, // JSONB containment temporal: { // time-based filter contains: "2025-06-15T00:00:00Z", // point-in-time @@ -165,119 +179,124 @@ for (const { memory, score } of results) { } ``` -## Error handling +## Space management -The client throws `RpcError` for application errors. Each error has a numeric `code` and an optional string `appCode` for programmatic matching. - -```typescript -import { createClient, RpcError } from "@memory.build/client"; +The memory client also exposes the in-space management namespaces. These require the appropriate authority (admin for roster/groups/invites; `owner@path` for grants). See [Access Control](access-control.md). -try { - await me.memory.get({ id: "nonexistent" }); -} catch (error) { - if (error instanceof RpcError) { - console.error(error.message); // human-readable message +### principal — the roster - if (error.is("NOT_FOUND")) { - // handle missing memory - } - } -} +```typescript +const { principals } = await me.principal.list(); // admin only +await me.principal.add({ principalId: "019..." }); +await me.principal.remove({ principalId: "019..." }); +const { principals } = await me.principal.resolve({ name: "alice@example.com" }); // any member +const { principals } = await me.principal.lookup({ ids: ["019..."] }); // any member ``` -### Error codes +### group -| `appCode` | Meaning | -|-----------|---------| -| `NOT_FOUND` | Resource doesn't exist | -| `UNAUTHORIZED` | Missing or invalid API key | -| `FORBIDDEN` | Insufficient permissions | -| `CONFLICT` | Duplicate or conflicting operation | -| `RATE_LIMITED` | Too many requests | -| `VALIDATION_ERROR` | Invalid input | -| `EMBEDDING_NOT_CONFIGURED` | Semantic search without embedding provider | -| `EMBEDDING_FAILED` | Embedding generation failed | -| `INTERNAL_ERROR` | Server error | - -## Access control +```typescript +const group = await me.group.create({ name: "backend" }); +const { groups } = await me.group.list(); +await me.group.rename({ groupId: group.id, name: "backend-team" }); +await me.group.delete({ groupId: group.id }); +await me.group.addMember({ groupId: group.id, memberId: "019...", admin: false }); +await me.group.removeMember({ groupId: group.id, memberId: "019..." }); +const { members } = await me.group.listMembers({ groupId: group.id }); +const { groups } = await me.group.listForMember({ memberId: "019..." }); +``` -The client exposes namespaces for managing users, grants, roles, and owners. These mirror the [Access Control](access-control.md) system. +### grant — tree access -### Users +Levels are `1` (read), `2` (write), `3` (owner). ```typescript -const user = await me.user.create({ name: "alice" }); -const user = await me.user.get({ id: "019..." }); -const user = await me.user.getByName({ name: "alice" }); -const { users } = await me.user.list(); -await me.user.rename({ id: "019...", name: "bob" }); -await me.user.delete({ id: "019..." }); +await me.grant.set({ principalId: "019...", treePath: "share.work", access: 2 }); +await me.grant.remove({ principalId: "019...", treePath: "share.work" }); +const { grants } = await me.grant.list(); // optionally { principalId } / { treePath } ``` -### Grants +### invite ```typescript -await me.grant.create({ - userId: "019...", - treePath: "team.shared", - actions: ["read", "create"], - withGrantOption: false, -}); -const { grants } = await me.grant.list({ userId: "019..." }); -const { allowed } = await me.grant.check({ - userId: "019...", - treePath: "team.shared", - action: "create", -}); -await me.grant.revoke({ userId: "019...", treePath: "team.shared" }); +// shareAccess is a level number (1=read, 2=write, 3=owner) at the shared root; null/omit = none +const invite = await me.invite.create({ email: "alice@example.com", admin: false, shareAccess: 1 }); +const { invitations } = await me.invite.list(); +await me.invite.revoke({ email: "alice@example.com" }); ``` -### Roles +## User-scoped operations + +Use `createUserClient` (session token only) for identity, agents, API keys, and space discovery. ```typescript -const role = await me.role.create({ name: "editors" }); -await me.role.addMember({ roleId: role.id, memberId: userId }); -await me.role.removeMember({ roleId: role.id, memberId: userId }); -const { members } = await me.role.listMembers({ roleId: role.id }); -const { roles } = await me.role.listForUser({ userId }); -``` +import { createUserClient } from "@memory.build/client"; -### Owners +const user = createUserClient({ token: sessionToken }); -```typescript -await me.owner.set({ userId: "019...", treePath: "team.shared" }); -const owner = await me.owner.get({ treePath: "team.shared" }); -const { owners } = await me.owner.list(); -await me.owner.remove({ treePath: "team.shared" }); -``` +// Identity +const me = await user.whoami(); -### API keys +// Spaces — discover and manage the spaces you belong to +const { spaces } = await user.space.list(); +const space = await user.space.create({ name: "My Space" }); // → { id, slug } +await user.space.rename({ slug: space.slug, name: "Renamed" }); +await user.space.delete({ slug: space.slug }); -```typescript -const { apiKey, rawKey } = await me.apiKey.create({ - userId: "019...", +// Agents — your global service accounts +const agent = await user.agent.create({ name: "ci-bot" }); // → { id } +const { agents } = await user.agent.list(); +await user.agent.rename({ id: agent.id, name: "ci-runner" }); +await user.agent.delete({ id: agent.id }); + +// API keys — global per-agent credentials +const { id, key } = await user.apiKey.create({ + agentId: agent.id, name: "ci-pipeline", expiresAt: "2026-01-01T00:00:00Z", // optional }); -console.log(rawKey); // "me.xxx.yyy" — only shown once - -const { apiKeys } = await me.apiKey.list({ userId: "019..." }); -await me.apiKey.revoke({ id: apiKey.id }); -await me.apiKey.delete({ id: apiKey.id }); +console.log(key); // "me.xxx.yyy" — full key returned once; only its hash is stored +const { apiKeys } = await user.apiKey.list({ memberId: agent.id }); +const apiKeyMeta = await user.apiKey.get({ id }); +await user.apiKey.delete({ id }); ``` -## Protocol +API keys are **global** per-agent credentials, not bound to a space: the same key works in any space the agent has been admitted to (the space comes from `X-Me-Space`). -The client communicates over JSON-RPC 2.0 via a single HTTP endpoint (`POST /api/v1/engine/rpc`). Authentication is via `Authorization: Bearer ` header. +## Error handling -You can make raw RPC calls using the `call` method: +The client throws `RpcError` for application errors. Each error has a numeric `code` and an optional string `appCode` for programmatic matching. ```typescript -const result = await me.call("memory.search", { semantic: "hello" }); +import { createMemoryClient, RpcError } from "@memory.build/client"; + +try { + await me.memory.get({ id: "nonexistent" }); +} catch (error) { + if (error instanceof RpcError) { + console.error(error.message); // human-readable message + if (error.is("NOT_FOUND")) { + // handle missing memory + } + } +} ``` -The API key can be swapped at runtime: +### Error codes -```typescript -me.setApiKey("me.newkey.newsecret"); -``` +| `appCode` | Meaning | +|-----------|---------| +| `NOT_FOUND` | Resource doesn't exist (or not visible to you) | +| `UNAUTHORIZED` | Missing or invalid session token / API key | +| `FORBIDDEN` | Insufficient permissions | +| `CONFLICT` | Duplicate or conflicting operation | +| `LAST_ADMIN` | Operation would leave the space with no effective admin | +| `RATE_LIMITED` | Too many requests | +| `VALIDATION_ERROR` | Invalid input | +| `EMBEDDING_NOT_CONFIGURED` | Semantic search without embedding provider | +| `EMBEDDING_FAILED` | Embedding generation failed | +| `INTERNAL_ERROR` | Server error | + +## Protocol + +Both clients speak JSON-RPC 2.0 over HTTP. The memory client uses `POST /api/v1/memory/rpc` with `Authorization: Bearer ` and a required `X-Me-Space: ` header; the user client uses `POST /api/v1/user/rpc` with a session-token bearer. See the [Access Control](access-control.md) guide for the authority model behind the management namespaces. diff --git a/packages/docs-site/lib/nav.ts b/packages/docs-site/lib/nav.ts index 9be813c..51d8593 100644 --- a/packages/docs-site/lib/nav.ts +++ b/packages/docs-site/lib/nav.ts @@ -32,7 +32,7 @@ export const NAV: NavSection[] = [ { label: "me login", slug: "cli/me-login" }, { label: "me logout", slug: "cli/me-logout" }, { label: "me whoami", slug: "cli/me-whoami" }, - { label: "me engine", slug: "cli/me-engine" }, + { label: "me space", slug: "cli/me-space" }, { label: "me memory", slug: "cli/me-memory" }, { label: "me mcp", slug: "cli/me-mcp" }, { label: "me claude", slug: "cli/me-claude" }, @@ -41,13 +41,10 @@ export const NAV: NavSection[] = [ { label: "me opencode", slug: "cli/me-opencode" }, { label: "me serve", slug: "cli/me-serve" }, { label: "Agent session imports", slug: "cli/agent-session-imports" }, - { label: "me user", slug: "cli/me-user" }, - { label: "me role", slug: "cli/me-role" }, - { label: "me grant", slug: "cli/me-grant" }, - { label: "me owner", slug: "cli/me-owner" }, - { label: "me org", slug: "cli/me-org" }, - { label: "me invitation", slug: "cli/me-invitation" }, + { label: "me agent", slug: "cli/me-agent" }, { label: "me apikey", slug: "cli/me-apikey" }, + { label: "me group", slug: "cli/me-group" }, + { label: "me access", slug: "cli/me-access" }, { label: "me pack", slug: "cli/me-pack" }, { label: "me completions", slug: "cli/me-completions" }, ], From 686cd4563edbb77297d57ea79fa0613cc34af56b Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Tue, 9 Jun 2026 14:33:54 +0200 Subject: [PATCH 120/156] fix(server): update Dockerfile for the auth rename + new workspaces The Dockerfile still copied packages/accounts (renamed to auth) and was missing the database and e2e workspaces, so bun install --frozen-lockfile failed in the dev build. Also drop the runtime copy of client, which is not a transitive dependency of the server. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/server/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 1f5a878..f2e01b0 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -9,9 +9,10 @@ EXPOSE 3000 # workspace declared in bun.lock is missing from the build context, so any # new workspace added under packages/* must be copied here too. COPY package.json bun.lock ./ -COPY packages/accounts/package.json packages/accounts/ +COPY packages/auth/package.json packages/auth/ COPY packages/cli/package.json packages/cli/ COPY packages/client/package.json packages/client/ +COPY packages/database/package.json packages/database/ COPY packages/docs-site/package.json packages/docs-site/ COPY packages/embedding/package.json packages/embedding/ COPY packages/engine/package.json packages/engine/ @@ -20,6 +21,7 @@ COPY packages/server/package.json packages/server/ COPY packages/web/package.json packages/web/ COPY packages/worker/package.json packages/worker/ COPY scripts/package.json scripts/ +COPY e2e/package.json e2e/ # --filter limits installation to the server and its transitive workspace deps, # avoiding pulling in docs-site's heavy runtime deps (next/react/tailwind/...). @@ -32,8 +34,8 @@ COPY version.ts ./ # Copy server source + all workspace dependencies COPY packages/server/ packages/server/ -COPY packages/accounts/ packages/accounts/ -COPY packages/client/ packages/client/ +COPY packages/auth/ packages/auth/ +COPY packages/database/ packages/database/ COPY packages/engine/ packages/engine/ COPY packages/embedding/ packages/embedding/ COPY packages/protocol/ packages/protocol/ From 8db64570aa2ae855a37c20df3864802625521cd7 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 10:28:41 +0200 Subject: [PATCH 121/156] fix(server): temporarily fall back to ENGINE_DATABASE_URL The dev deploy injects the legacy two-DB env (ACCOUNTS_DATABASE_URL / ENGINE_DATABASE_URL); the single-DB code requires DATABASE_URL and crashes on boot. Fall back to ENGINE_DATABASE_URL so multiplayer can deploy to dev before the tiger-agents-deploy helm values are migrated to the single-DB env contract. Remove once the deploy config sets DATABASE_URL. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/server/start.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/server/start.ts b/packages/server/start.ts index b53f1ec..268bd93 100644 --- a/packages/server/start.ts +++ b/packages/server/start.ts @@ -139,9 +139,18 @@ export async function startServer( const port = opts.port ?? (process.env.PORT ? Number(process.env.PORT) : 3000); - const databaseUrl = opts.databaseUrl ?? process.env.DATABASE_URL; + // TEMPORARY: fall back to the legacy ENGINE_DATABASE_URL so the multiplayer + // branch can deploy to dev before the tiger-agents-deploy helm values are + // migrated to the single-DB env contract. Remove once the deploy config sets + // DATABASE_URL directly. + const databaseUrl = + opts.databaseUrl ?? + process.env.DATABASE_URL ?? + process.env.ENGINE_DATABASE_URL; if (!databaseUrl) { - throw new Error("DATABASE_URL environment variable is required"); + throw new Error( + "DATABASE_URL (or legacy ENGINE_DATABASE_URL) environment variable is required", + ); } const apiBaseUrl = opts.apiBaseUrl ?? process.env.API_BASE_URL; From 1cc14f603f2e10d2755b838533062e7ab077bcfb Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:07:12 +0200 Subject: [PATCH 122/156] feat(cli): me claude install drives the full plugin by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `me claude install` now installs the Memory Engine plugin (hooks + slash commands + MCP) by driving Claude Code's native plugin CLI for you — `marketplace add` (idempotent) + `plugin install`, passing the resolved server/space/api_key through --config. `--mcp-only` falls back to the previous behavior (register just the `me` MCP server). `--dev` installs from the local checkout's .claude-plugin/marketplace.json (clean swap, since the local and published marketplaces share the "memory-engine" name) so captures exercise your working tree. Docs updated across README, getting-started, mcp-integration, agents.txt, and the me-claude CLI reference to reflect the full-plugin default. Co-Authored-By: Claude Opus 4.8 (1M context) --- DEVELOPMENT.md | 8 + README.md | 6 +- docs/agents.txt | 6 +- docs/cli/me-claude.md | 37 +++-- docs/getting-started.md | 8 +- docs/mcp-integration.md | 11 +- packages/cli/commands/claude.ts | 284 ++++++++++++++++++++++++++++++-- 7 files changed, 317 insertions(+), 43 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3425fcf..327f682 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -5,6 +5,14 @@ - [Bun](https://bun.sh) (latest) - [Docker](https://www.docker.com/) (for PostgreSQL) +## Quick Start against dev + +```bash +./ bun install:local +me --server https://me.dev-us-east-1.ops.dev.timescale.com login +me claude install +``` + ## Quick Start ```bash diff --git a/README.md b/README.md index 5abdcbb..4022169 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,7 @@ me memory search "how does authentication work" me opencode install me codex install me gemini install -me claude install # MCP-only - -# Or, for the full Claude Code plugin (hooks + slash commands + MCP): -claude plugin marketplace add timescale/memory-engine -claude plugin install memory-engine@memory-engine +me claude install # full plugin: hooks + slash commands + MCP ``` ## How it works diff --git a/docs/agents.txt b/docs/agents.txt index f95e6e0..1ab9b7e 100644 --- a/docs/agents.txt +++ b/docs/agents.txt @@ -14,9 +14,9 @@ mcp: opencode: me opencode install codex_cli: me codex install gemini_cli: me gemini install - claude_code: me claude install - claude_code_plugin: claude plugin marketplace add timescale/memory-engine && claude plugin install memory-engine@memory-engine - description: Memory engine ships as an MCP server. OpenCode, Codex CLI, Gemini CLI, and Claude Code all support MCP-only install via `me install`. Claude Code additionally supports a full plugin (hooks + slash commands + MCP) via Claude Code's plugin marketplace. + claude_code: me claude install --mcp-only + claude_code_plugin: me claude install + description: Memory engine ships as an MCP server. OpenCode, Codex CLI, Gemini CLI, and Claude Code all support MCP-only install via `me install` (Claude Code: `me claude install --mcp-only`). For Claude Code, the default `me claude install` installs the full plugin (hooks + slash commands + MCP) via Claude Code's plugin marketplace. compatible_clients: - Claude Code - Codex CLI diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index bad68c4..80d7630 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -4,7 +4,7 @@ Claude Code integration commands. ## Commands -- [me claude install](#me-claude-install) -- register `me` as an MCP server with Claude Code (MCP-only) +- [me claude install](#me-claude-install) -- install the Memory Engine plugin for Claude Code (full plugin by default, `--mcp-only` for just the MCP server) - [me claude hook](#me-claude-hook) -- invoked by the Claude Code plugin to capture events as memories - [me claude import](#me-claude-import) -- import Claude Code sessions from `~/.claude/projects` @@ -12,28 +12,41 @@ Claude Code integration commands. ## me claude install -Register `me` as an MCP server with Claude Code. +Install the Memory Engine plugin for Claude Code. -This is the **MCP-only** install path: it adds the `me` tools to Claude Code without installing the full Memory Engine plugin. If you want hooks (auto-capture of Claude Code events) and slash commands, install the plugin instead -- see [me claude hook](#me-claude-hook). +By default this installs the **full plugin** -- hooks (auto-capture of Claude Code events), slash commands, and the MCP tools -- by driving Claude Code's native plugin CLI for you: ``` me claude install [options] ``` +Under the hood it runs the equivalent of: + +```bash +claude plugin marketplace add timescale/memory-engine +claude plugin install memory-engine@memory-engine \ + --config server= [--config space=] [--config api_key=] +``` + +The marketplace step is idempotent (skipped if already configured), and the resolved `server` / `space` / `api_key` are passed through `--config` -- the same path as the interactive `/plugin` configure flow. After install, restart Claude Code (or run `/plugin`) to load the hooks and slash commands. + +Pass `--mcp-only` to skip the plugin and register just the `me` MCP server (no hooks, no slash commands -- the previous default behavior). + | Option | Description | |--------|-------------| -| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--mcp-only` | Register only the `me` MCP server (no hooks or slash commands). | +| `--api-key ` | API key for a headless agent. Default: the plugin/MCP server uses your `me login` session, resolved at runtime. | | `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | -| `--server ` | Server URL to embed in the MCP config. | +| `--server ` | Server URL to embed in the config. | | `-s, --scope ` | Claude Code config scope: `local`, `user`, or `project`. Default: `user`. | -By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. +Credential handling is the same for both modes: with no `--api-key`, the plugin (and the MCP server) uses your `me login` session, resolved from the OS keychain / `~/.config/me` at runtime (so it survives re-login), and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that requires a pinned `--space`. -The `--scope` flag mirrors `claude mcp add --scope`: +The `--scope` flag mirrors `claude plugin install --scope` / `claude mcp add --scope`: -- `local` -- registration scoped to the current project on this machine only. -- `user` -- registration available to all projects for your user (default). -- `project` -- registration committed to the current project (e.g. checked into `.claude/`). +- `local` -- scoped to the current project on this machine only. +- `user` -- available to all projects for your user (default). +- `project` -- committed to the current project (e.g. checked into `.claude/`). For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). @@ -51,7 +64,7 @@ me claude hook --event |--------|-------------| | `--event ` | Hook event name (required). | -This command is not run directly -- the Claude Code plugin calls it. The plugin (which includes hooks, slash commands, and MCP) is installed via Claude Code's native flow: +This command is not run directly -- the Claude Code plugin calls it. The plugin (which includes hooks, slash commands, and MCP) is installed by [me claude install](#me-claude-install), which drives Claude Code's native plugin flow for you. You can also run that flow by hand: ```bash claude plugin marketplace add timescale/memory-engine @@ -62,7 +75,7 @@ claude plugin install memory-engine@memory-engine [--scope user|project|local] Both `api_key` and `space` are optional: blank `api_key` uses your `me login` session (set it to attribute captures to a dedicated agent), and blank `space` uses your active space (`me space use`; pin it for project/shared installs). -If you only want the MCP tools (no hooks, no slash commands), use [me claude install](#me-claude-install) instead. +If you only want the MCP tools (no hooks, no slash commands), run [me claude install --mcp-only](#me-claude-install) instead. Best-effort: logs failures to stderr but always exits 0 so that a hook failure never blocks a Claude Code session. diff --git a/docs/getting-started.md b/docs/getting-started.md index 9256fc8..4f6ee62 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -83,14 +83,14 @@ me codex install me gemini install ``` -For Claude Code, install the Memory Engine plugin instead: +For Claude Code, `me claude install` installs the full Memory Engine plugin (hooks + slash commands + MCP): ```bash -claude plugin marketplace add timescale/memory-engine -claude plugin install memory-engine@memory-engine +me claude install # full plugin +me claude install --mcp-only # or just the MCP server ``` -Then start Claude Code, run `/plugin`, select `memory-engine`, and configure the options. All are optional except `server`: leave `api_key` blank to use your `me login` session, leave `space` blank to use your active space, and `tree_root` defaults to `share.projects`. +This drives Claude Code's native plugin flow for you (`claude plugin marketplace add` + `claude plugin install`), passing your resolved server/space/api_key through `--config`. Afterwards, restart Claude Code (or run `/plugin`) to load the hooks and slash commands; you can re-run `/plugin` → `memory-engine` → Configure to adjust options. All are optional except `server`: leave `api_key` blank to use your `me login` session, leave `space` blank to use your active space, and `tree_root` defaults to `share.projects`. After installation, your AI agent has access to memory tools -- create, search, get, update, delete, and more. diff --git a/docs/mcp-integration.md b/docs/mcp-integration.md index 8a134b0..cd3fac9 100644 --- a/docs/mcp-integration.md +++ b/docs/mcp-integration.md @@ -41,16 +41,23 @@ See the agent-specific command references for details: [`me opencode install`](c | OpenCode | `me opencode install` | | Codex CLI | `me codex install` | | Gemini CLI | `me gemini install` | -| Claude Code | Claude Code plugin, described below | +| Claude Code | `me claude install` (full plugin) / `me claude install --mcp-only` | ### Claude Code +```bash +me claude install # full plugin: hooks + slash commands + MCP +me claude install --mcp-only # or just the MCP server +``` + +By default `me claude install` installs the Memory Engine plugin, driving Claude Code's native plugin flow for you (`claude plugin marketplace add` + `claude plugin install`) and passing your resolved `server` / `space` / `api_key` through `--config`. The plugin provides the MCP server and captures Claude Code session events as memories. After installing, restart Claude Code (or run `/plugin`) to load the hooks and slash commands; re-run `/plugin` → `memory-engine` → Configure to adjust options. To run the underlying flow by hand instead: + ```bash claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] ``` -Claude Code uses the Memory Engine plugin. After installing it, start a Claude Code session, run `/plugin`, select `memory-engine`, and configure `space` (and optionally `api_key`, `server`, `tree_root`). The plugin provides the MCP server and captures Claude Code session events as memories. +See [`me claude install`](cli/me-claude.md#me-claude-install) for the full option reference. ### Gemini CLI diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index bd005bc..2f5d2d6 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -1,24 +1,34 @@ /** * me claude — Claude Code integration commands. * - * Two integration paths: + * `me claude install` has two modes: * - * 1. Full plugin (hooks + slash commands + MCP) via Claude Code's native - * plugin marketplace: + * 1. Full plugin (default) — installs the Memory Engine plugin (hooks + + * slash commands + MCP) via Claude Code's native plugin marketplace, + * driving the same commands you'd otherwise run by hand: * * claude plugin marketplace add timescale/memory-engine - * claude plugin install memory-engine@memory-engine [--scope user|project|local] - * # then, in a Claude Code session: - * /plugin # select memory-engine, Configure, fill space (+ optional api_key) + * claude plugin install memory-engine@memory-engine \ + * --config server=… [--config space=…] [--config api_key=…] * * Claude Code delivers the configured values to our hook (`me claude * hook --event `) via CLAUDE_PLUGIN_OPTION_* env vars. api_key is * optional: left blank, the hook (and the plugin's MCP server) use your * `me login` session. * - * 2. MCP-only via `me claude install`. Registers `me` as an MCP server - * with Claude Code (no hooks, no slash commands — just the tools). + * Pass --dev (run from inside the repo) to install the plugin from your + * local checkout — the repo's .claude-plugin/marketplace.json — instead of + * the published marketplace. The two share the marketplace name + * "memory-engine", so --dev re-points it at your working tree and + * reinstalls fresh (plugin files are copied into the cache, so a new build + * needs a reinstall). + * + * 2. MCP-only (`--mcp-only`) — registers `me` as an MCP server with Claude + * Code (no hooks, no slash commands — just the tools). */ +import { existsSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import * as clack from "@clack/prompts"; import { Command, InvalidArgumentError } from "commander"; import { HOOK_EVENT_NAMES, @@ -37,6 +47,13 @@ import { } from "../mcp/agent-install.ts"; import { buildAgentImportSubcommand } from "./import.ts"; +/** GitHub source for `claude plugin marketplace add`. */ +const PLUGIN_MARKETPLACE_SOURCE = "timescale/memory-engine"; +/** The marketplace `name` (from .claude-plugin/marketplace.json). */ +const PLUGIN_MARKETPLACE_NAME = "memory-engine"; +/** `@` ref for `claude plugin install`. */ +const PLUGIN_REF = `memory-engine@${PLUGIN_MARKETPLACE_NAME}`; + const CLAUDE_SCOPES = ["local", "user", "project"] as const; type ClaudeScope = (typeof CLAUDE_SCOPES)[number]; @@ -50,20 +67,26 @@ function parseClaudeScope(value: string): ClaudeScope { } /** - * me claude install — register me as an MCP server with Claude Code. + * me claude install — install the Memory Engine plugin for Claude Code. * - * MCP-only: leaves the full Claude Code plugin install flow alone. Use this - * if you want the `me` MCP tools available in Claude Code but don't want the - * plugin's hooks or slash commands. + * Default: the full plugin (hooks + slash commands + MCP), installed via + * Claude Code's native plugin marketplace. `--mcp-only` falls back to + * registering just the `me` MCP server (no hooks, no slash commands). */ function createClaudeInstallCommand(): Command { return new Command("install") - .description("register me as an MCP server with Claude Code") + .description( + "install the Memory Engine plugin for Claude Code (hooks + slash commands + MCP)", + ) + .option( + "--mcp-only", + "register only the me MCP server (no hooks or slash commands)", + ) .option( "--api-key ", "API key for a headless agent (default: use your login session at runtime)", ) - .option("--server ", "server URL to embed in MCP config") + .option("--server ", "server URL to embed in the config") .option( "--space ", "pin a space (default: resolve ME_SPACE / active space at runtime)", @@ -74,22 +97,249 @@ function createClaudeInstallCommand(): Command { parseClaudeScope, "user", ) + .option( + "--dev", + "install the plugin from the local checkout instead of the published marketplace (run from inside the repo)", + ) .action( async ( - opts: AgentInstallOptions & { scope: ClaudeScope }, + opts: AgentInstallOptions & { + scope: ClaudeScope; + mcpOnly?: boolean; + dev?: boolean; + }, cmd: Command, ) => { const globalOpts = cmd.optsWithGlobals(); - await runAgentMcpInstall("claude", { + const server = globalOpts.server ?? opts.server; + if (opts.mcpOnly) { + if (opts.dev) { + clack.log.warn( + "--dev has no effect with --mcp-only: the MCP server already runs your local `me` binary on PATH.", + ); + } + await runAgentMcpInstall("claude", { + apiKey: opts.apiKey, + server, + space: opts.space, + scope: opts.scope, + }); + return; + } + await runClaudePluginInstall({ apiKey: opts.apiKey, - server: globalOpts.server ?? opts.server, + server, space: opts.space, scope: opts.scope, + dev: opts.dev, }); }, ); } +/** Run a command, capturing its exit code, stdout, and stderr. */ +async function runCommand( + cmd: string[], +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const exitCode = await proc.exited; + return { exitCode, stdout, stderr }; +} + +/** + * Walk up from `startDir` to the repo's marketplace manifest + * (`.claude-plugin/marketplace.json`), returning the directory that contains it + * — the marketplace root passed to `claude plugin marketplace add`. Used by + * `--dev` to install the plugin from the local checkout. Returns undefined when + * not run from inside the repo. + */ +function findRepoMarketplaceRoot(startDir: string): string | undefined { + let dir = resolve(startDir); + for (;;) { + if (existsSync(join(dir, ".claude-plugin", "marketplace.json"))) return dir; + const parent = dirname(dir); + if (parent === dir) return undefined; // reached filesystem root + dir = parent; + } +} + +/** + * Install the full Memory Engine plugin for Claude Code. + * + * Drives Claude Code's plugin CLI: registers the marketplace (idempotent — a + * no-op if it's already configured) and installs the plugin, passing the + * resolved server/space/api_key through `--config` (the same path as the + * interactive `/plugin` configure flow). Credential handling mirrors the + * MCP-only path: an api key requires a pinned space; otherwise the plugin + * falls back to your `me login` session at runtime. + */ +async function runClaudePluginInstall( + opts: AgentInstallOptions & { scope: ClaudeScope; dev?: boolean }, +): Promise { + if (Bun.which("claude") === null) { + clack.log.error( + "Claude Code (claude) not found on PATH. Install it first.", + ); + process.exit(1); + } + + // Resolve credentials: flags > env (ME_API_KEY / ME_SERVER / ME_SPACE) > + // stored config. + const creds = resolveCredentials(opts.server); + const apiKey = opts.apiKey ?? creds.apiKey; + const server = opts.server ?? creds.server; + if (!server) { + clack.log.error("No server URL available. Pass --server or set ME_SERVER."); + process.exit(1); + } + const space = opts.space ?? creds.activeSpace; + + if (apiKey) { + // A global key isn't space-bound, so the space must be fixed. + if (!space) { + clack.log.error( + "No space for the API key. Pass --space, set ME_SPACE, or run 'me space use ' (keys are global, so the space must be fixed).", + ); + process.exit(1); + } + } else if (!creds.sessionToken) { + clack.log.error( + "Not logged in. Run 'me login' (the plugin will use your session), or pass --api-key / set ME_API_KEY for a headless agent.", + ); + process.exit(1); + } else if (!space) { + clack.log.warn( + "No active space set — captures are skipped until you run 'me space use ' (or set ME_SPACE). Re-run with --space to pin one.", + ); + } + + // Resolve the marketplace source: the published GitHub repo, or — with --dev + // — the local checkout, so captures exercise the plugin files from your + // working tree (.mcp.json, hooks, slash commands) rather than the published + // version. + let marketplaceSource = PLUGIN_MARKETPLACE_SOURCE; + if (opts.dev) { + const root = findRepoMarketplaceRoot(process.cwd()); + if (!root) { + clack.log.error( + "--dev must be run from inside the memory-engine repo (no .claude-plugin/marketplace.json found at or above the current directory).", + ); + process.exit(1); + } + marketplaceSource = root; + } + + const spin = clack.spinner(); + + // 1. Register the marketplace. + if (opts.dev) { + // The local and published marketplaces share the name "memory-engine", so + // they can't coexist and `marketplace add` won't re-point an existing name; + // plugin install also copies files into the cache, so a fresh build needs a + // reinstall. Tear both down first (ignoring "not found" — these may be a + // no-op on a clean machine), then re-add from the local checkout so the + // install below picks up your working tree. + spin.start( + "Pointing the Memory Engine marketplace at your local checkout...", + ); + await runCommand([ + "claude", + "plugin", + "uninstall", + "-y", + "-s", + opts.scope, + PLUGIN_REF, + ]); + await runCommand([ + "claude", + "plugin", + "marketplace", + "remove", + PLUGIN_MARKETPLACE_NAME, + ]); + const add = await runCommand([ + "claude", + "plugin", + "marketplace", + "add", + "--scope", + opts.scope, + marketplaceSource, + ]); + if (add.exitCode !== 0 && !/already/i.test(add.stderr + add.stdout)) { + spin.stop("Failed to add the local marketplace"); + clack.log.error( + `claude plugin marketplace add exited with ${add.exitCode}${add.stderr ? ` — ${add.stderr.trim()}` : ""}`, + ); + process.exit(1); + } + } else { + // Idempotent: skip if already there. + spin.start("Adding the Memory Engine marketplace..."); + const list = await runCommand(["claude", "plugin", "marketplace", "list"]); + const alreadyAdded = + list.exitCode === 0 && list.stdout.includes(marketplaceSource); + if (!alreadyAdded) { + const add = await runCommand([ + "claude", + "plugin", + "marketplace", + "add", + "--scope", + opts.scope, + marketplaceSource, + ]); + if (add.exitCode !== 0 && !/already/i.test(add.stderr + add.stdout)) { + spin.stop("Failed to add the marketplace"); + clack.log.error( + `claude plugin marketplace add exited with ${add.exitCode}${add.stderr ? ` — ${add.stderr.trim()}` : ""}`, + ); + process.exit(1); + } + } + } + + // 2. Install the plugin, baking the resolved config so captures land in the + // right space. Leave tree_root / content_mode at the plugin defaults + // (reconfigure them later via `/plugin` if needed). + spin.message("Installing the memory-engine plugin..."); + const install = ["claude", "plugin", "install", "--scope", opts.scope]; + install.push("--config", `server=${server}`); + if (space) install.push("--config", `space=${space}`); + if (apiKey) install.push("--config", `api_key=${apiKey}`); + install.push(PLUGIN_REF); + + const result = await runCommand(install); + if (result.exitCode !== 0) { + if (/already/i.test(result.stderr + result.stdout)) { + spin.stop("Memory Engine plugin already installed"); + clack.log.info( + "Run '/plugin' in Claude Code to reconfigure (or '--mcp-only' for the MCP server alone).", + ); + return; + } + spin.stop("Failed to install the plugin"); + clack.log.error( + `claude plugin install exited with ${result.exitCode}${result.stderr ? ` — ${result.stderr.trim()}` : ""}`, + ); + process.exit(1); + } + + spin.stop( + opts.dev + ? "Installed the Memory Engine plugin from your local checkout" + : "Installed the Memory Engine plugin for Claude Code", + ); + clack.log.info( + "Restart Claude Code (or run '/plugin') to load the hooks + slash commands.", + ); +} + /** * me claude hook — invoked by the Claude Code plugin on Stop / SessionEnd to * capture the session. From c3634e7b4c82b7b01a51f2a6a1af59a83863f08a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:16:12 +0200 Subject: [PATCH 123/156] fix(server): route memory.search tree filter to lquery/ltxtquery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit memory.search always cast the tree filter to ::ltree, so a documented lquery (`foo.*`) or ltxtquery (`a & b`) threw `invalid ltree` — surfaced to callers as a generic Internal error. Only a bare path worked. Add classifyTreeFilter() in space/path.ts: it normalizes (~/slashes) then classifies the result — bare path -> ltree containment, contains `&` -> ltxtquery, else -> lquery — and the search handler binds the matching SQL parameter (@> / ~ / @). The store already accepted all three; only the handler was collapsing everything to ltree. Tests: classifier unit cases in path.test.ts, plus end-to-end lquery wildcard and ltxtquery cases in memory.integration.test.ts that exercise the real SQL and would have caught the original crash. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/database/space/index.ts | 2 + packages/database/space/path.test.ts | 58 +++++++++++++++++++ packages/database/space/path.ts | 37 ++++++++++++ .../rpc/memory/memory.integration.test.ts | 34 +++++++++++ packages/server/rpc/memory/memory.ts | 10 +++- packages/server/rpc/memory/support.ts | 17 ++++-- 6 files changed, 151 insertions(+), 7 deletions(-) diff --git a/packages/database/space/index.ts b/packages/database/space/index.ts index ff19100..1181e10 100644 --- a/packages/database/space/index.ts +++ b/packages/database/space/index.ts @@ -5,12 +5,14 @@ export { provisionSpace, } from "./migrate/migrate"; export { + classifyTreeFilter, denormalizeTreePath, HOME_NAMESPACE, homePrefix, normalizeTreeFilter, normalizeTreePath, SHARE_NAMESPACE, + type TreeFilter, TreePathError, type TreePathOptions, } from "./path"; diff --git a/packages/database/space/path.test.ts b/packages/database/space/path.test.ts index 9465221..2755253 100644 --- a/packages/database/space/path.test.ts +++ b/packages/database/space/path.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { + classifyTreeFilter, denormalizeTreePath, homePrefix, normalizeTreeFilter, @@ -77,6 +78,63 @@ describe("normalizeTreeFilter", () => { }); }); +describe("classifyTreeFilter", () => { + test("empty input is no filter", () => { + expect(classifyTreeFilter("")).toBeNull(); + expect(classifyTreeFilter("/")).toBeNull(); + expect(classifyTreeFilter(" ")).toBeNull(); + }); + + test("a bare path classifies as ltree (containment)", () => { + expect(classifyTreeFilter("share")).toEqual({ + kind: "ltree", + value: "share", + }); + expect(classifyTreeFilter("/share/projects/")).toEqual({ + kind: "ltree", + value: "share.projects", + }); + expect(classifyTreeFilter("my-proj.notes_2")).toEqual({ + kind: "ltree", + value: "my-proj.notes_2", + }); + }); + + test("a wildcard classifies as lquery", () => { + expect(classifyTreeFilter("share.projects.*")).toEqual({ + kind: "lquery", + value: "share.projects.*", + }); + expect(classifyTreeFilter("*.api.*")).toEqual({ + kind: "lquery", + value: "*.api.*", + }); + // `|` and `!` are lquery label operators, not ltxtquery here. + expect(classifyTreeFilter("foo|bar.baz")).toEqual({ + kind: "lquery", + value: "foo|bar.baz", + }); + }); + + test("an `&` boolean classifies as ltxtquery", () => { + expect(classifyTreeFilter("api & v2")).toEqual({ + kind: "ltxtquery", + value: "api & v2", + }); + }); + + test("a leading ~ expands before classification", () => { + expect(classifyTreeFilter("~.*", { home: ID })).toEqual({ + kind: "lquery", + value: `${HOME}.*`, + }); + expect(classifyTreeFilter("~/notes", { home: ID })).toEqual({ + kind: "ltree", + value: `${HOME}.notes`, + }); + }); +}); + describe("homePrefix", () => { test("strips hyphens from the principal id", () => { expect(homePrefix(ID)).toBe(HOME); diff --git a/packages/database/space/path.ts b/packages/database/space/path.ts index 9d0b162..44c7802 100644 --- a/packages/database/space/path.ts +++ b/packages/database/space/path.ts @@ -137,6 +137,43 @@ export function normalizeTreeFilter( return s; } +/** A bare ltree path: dot-separated `[A-Za-z0-9_-]` labels, no query operators. */ +const LTREE_PATH = /^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$/; + +/** + * A classified, normalized search tree filter, tagged by which ltree + * pattern type it is so the caller can bind it to the matching SQL parameter + * (`ltree` → `@>` containment, `lquery` → `~`, `ltxtquery` → `@`). + */ +export type TreeFilter = + | { kind: "ltree"; value: string } + | { kind: "lquery"; value: string } + | { kind: "ltxtquery"; value: string }; + +/** + * Normalize a search tree filter (via `normalizeTreeFilter`) and classify it as + * an exact ltree path, an `lquery` pattern, or an `ltxtquery` label search. + * `normalizeTreeFilter` only expands `~`/slashes — it does not pick a type — so + * without this the caller can't know which SQL parameter to bind, and casting a + * wildcard like `foo.*` to `::ltree` throws. Returns `null` for empty input (no + * filter). + * + * Classification (the input has already had `~`/slashes normalized): + * - bare ltree path (only `[A-Za-z0-9_-]` labels + `.`) → `ltree` (containment) + * - contains `&` (ltxtquery's boolean AND — never valid in lquery) → `ltxtquery` + * - anything else (wildcards `*`, `|`, `!`, `{n}`, …) → `lquery` + */ +export function classifyTreeFilter( + input: string, + opts: TreePathOptions = {}, +): TreeFilter | null { + const s = normalizeTreeFilter(input, opts); + if (s === "") return null; + if (LTREE_PATH.test(s)) return { kind: "ltree", value: s }; + if (s.includes("&")) return { kind: "ltxtquery", value: s }; + return { kind: "lquery", value: s }; +} + /** * Reverse of the home expansion, for display. A path under the given * principal's home is shown with a leading `~`, keeping the canonical dot diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index bfe54d3..cb714f7 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -380,6 +380,40 @@ test("search: tree filter only (no ranking) returns matches", async () => { expect(res.results[0]?.tree).toBe("share.scope.a"); }); +test("search: tree lquery wildcard matches descendants", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "direct child", tree: "share.proj.a" }, + { content: "deep descendant", tree: "share.proj.a.deep" }, + { content: "sibling root", tree: "share.proj" }, + { content: "elsewhere", tree: "share.other" }, + ], + }); + // `share.proj.*` is lquery (not a literal ltree) — it must bind to the + // lquery param, not cast to ::ltree (which would throw). lquery `*` matches + // zero-or-more labels, so it matches share.proj and everything under it, but + // not share.other. + const res = await call<{ results: { tree: string }[] }>("memory.search", { + tree: "share.proj.*", + }); + const trees = res.results.map((r) => r.tree).sort(); + expect(trees).toEqual(["share.proj", "share.proj.a", "share.proj.a.deep"]); +}); + +test("search: tree ltxtquery (label boolean) matches by label", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "both labels", tree: "share.alpha.beta" }, + { content: "one label", tree: "share.alpha" }, + ], + }); + // `alpha & beta` is ltxtquery — must bind to the ltxtquery param. + const res = await call<{ results: { tree: string }[] }>("memory.search", { + tree: "alpha & beta", + }); + expect(res.results.map((r) => r.tree)).toEqual(["share.alpha.beta"]); +}); + test("search: grep alone is rejected", async () => { await expectAppError( call("memory.search", { grep: "anything" }), diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 15ebcab..15dc7c0 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -348,10 +348,14 @@ async function memorySearch( ? 1 - params.semanticThreshold : undefined; + // Classify the tree filter so a wildcard (`foo.*`) binds to lquery and a + // boolean label search (`a & b`) to ltxtquery, rather than all casting to + // ltree (which throws on query syntax). + const treeFilter = params.tree ? inputTreeFilter(ctx, params.tree) : null; const filters = { - ltree: params.tree - ? inputTreeFilter(ctx, params.tree) || undefined - : undefined, + ltree: treeFilter?.kind === "ltree" ? treeFilter.value : undefined, + lquery: treeFilter?.kind === "lquery" ? treeFilter.value : undefined, + ltxtquery: treeFilter?.kind === "ltxtquery" ? treeFilter.value : undefined, metaContains: params.meta ?? undefined, regexp: params.grep ?? undefined, ...mapTemporalFilter(params.temporal), diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts index 8fb5e5a..ec7dce4 100644 --- a/packages/server/rpc/memory/support.ts +++ b/packages/server/rpc/memory/support.ts @@ -5,9 +5,10 @@ */ import { + classifyTreeFilter, denormalizeTreePath, - normalizeTreeFilter, normalizeTreePath, + type TreeFilter, TreePathError, } from "@memory.build/database"; import type { @@ -49,10 +50,18 @@ export function inputTreePath(ctx: SpaceRpcContext, raw: string): string { } } -/** Like `inputTreePath` but for a search filter (lquery/ltxtquery passes through). */ -export function inputTreeFilter(ctx: SpaceRpcContext, raw: string): string { +/** + * Like `inputTreePath` but for a search filter: normalizes `~`/slashes and + * classifies the result as an ltree path, an `lquery` pattern, or an + * `ltxtquery` label search, so the handler can bind the right SQL parameter. + * Returns `null` when there is no filter. + */ +export function inputTreeFilter( + ctx: SpaceRpcContext, + raw: string, +): TreeFilter | null { try { - return normalizeTreeFilter(raw, { home: ctx.principalId }); + return classifyTreeFilter(raw, { home: ctx.principalId }); } catch (e) { throw asValidationError(e); } From c054778a45fcd71c814ca307d2e582fc2721a18b Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:18:54 +0200 Subject: [PATCH 124/156] docs: add dev quick-start to DEVELOPMENT.md Document installing the local `me` binary and the `me claude install --dev` flow against the dev server. Co-Authored-By: Claude Opus 4.8 (1M context) --- DEVELOPMENT.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 327f682..f1eb368 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -5,12 +5,12 @@ - [Bun](https://bun.sh) (latest) - [Docker](https://www.docker.com/) (for PostgreSQL) -## Quick Start against dev +## Quick Start against dev ```bash -./ bun install:local -me --server https://me.dev-us-east-1.ops.dev.timescale.com login -me claude install +./bun run install:local +me --server https://me.dev-us-east-1.ops.dev.timescale.com login +me claude install --dev ``` ## Quick Start From 8616ba37fb51b87da23bfea6081c881f6630fb0c Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:32:24 +0200 Subject: [PATCH 125/156] test(e2e): cover import backfilling pre-hook-install work Adds an e2e test asserting that `me claude import` backfills Claude Code work that predates the capture hook: a transcript is written to disk with no hook firing, the hook then captures a separate live session, and a subsequent import pulls in the pre-install transcript without duplicating the live capture. Also adds a countBySession() helper. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cli.e2e.test.ts | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index f870df5..4f3f81b 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -207,6 +207,16 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( return (row?.n as number) ?? 0; } + // Count memories captured from a given source session id. + async function countBySession(sessionId: string): Promise { + const [row] = await sql.unsafe( + `select count(*)::int as n from metest_${spaceSlug}.memory + where meta->>'source_session_id' = $1`, + [sessionId], + ); + return (row?.n as number) ?? 0; + } + // Parse the --json stdout of a `me` invocation, asserting success. async function meJson( args: string[], @@ -353,6 +363,85 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(res.total).toBeGreaterThan(0); }); + test("8. `me claude import` backfills work that predates the hook", async () => { + // The scenario: a user does a bunch of Claude Code work BEFORE installing + // the capture hook (no hook fires for it), then installs the hook (which + // begins capturing new sessions live), then runs `me claude import`. The + // pre-install work must be backfilled — the importer has no lower time + // bound tied to hook install; it sweeps every transcript and dedupes by + // deterministic message id. + const root = await mkdtemp(join(tmpdir(), "me-e2e-backfill-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + + // cwd "/work/backfill-proj" → no git repo on disk → slug = basename, so + // both sessions land under the same tree. + const cwd = "/work/backfill-proj"; + const tree = "share.projects.backfill_proj.agent_sessions"; + + const mkMsg = + (sessionId: string) => + (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-02-01T00:00:0${i}.000Z`, + sessionId, + cwd, + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + + const writeTranscript = async (sessionId: string, prefix: string) => { + const m = mkMsg(sessionId); + const lines = [ + m(0, "user", `${prefix} first question`), + m(1, "assistant", `${prefix} first answer`), + m(2, "user", `${prefix} second question`), + m(3, "assistant", `${prefix} second answer`), + ]; + const path = join(projDir, `${sessionId}.jsonl`); + await writeFile(path, lines.map((l) => JSON.stringify(l)).join("\n")); + return path; + }; + + // 1. Pre-install work: a transcript sits on disk; NO hook ever fires for + // it. It must not be in the engine yet. + const oldSession = `pre-install-${rand()}`; + await writeTranscript(oldSession, "old"); + expect(await countBySession(oldSession)).toBe(0); + + // 2. Install the hook (it now captures live) and let it import a NEW + // session — the real `me claude hook` path, reading from stdin. + const newSession = `post-install-${rand()}`; + const newTranscript = await writeTranscript(newSession, "new"); + const hook = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ + transcript_path: newTranscript, + session_id: newSession, + }), + ); + expect(hook.code, hook.stderr).toBe(0); + // The hook captured only the post-install session — the old one is still + // absent (this is exactly the gap `me claude import` must close). + expect(await countBySession(newSession)).toBe(4); + expect(await countBySession(oldSession)).toBe(0); + + // 3. Run `me claude import`. + const imp = await me(["claude", "import", "--source", root]); + expect(imp.code, imp.stderr).toBe(0); + + // 4. The pre-install work is now backfilled, and the hook's live capture + // was not duplicated. + expect(await countBySession(oldSession)).toBe(4); + expect(await countBySession(newSession)).toBe(4); + expect(await countUnder(tree)).toBe(8); + + await rm(root, { recursive: true, force: true }); + }); + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { // A minimal Claude Code session transcript on disk. The importer scans // //*.jsonl; the hook reads the file directly. From 024d02ef0f95ec33d43722b592dd318bf231f8a2 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:37:16 +0200 Subject: [PATCH 126/156] test(e2e): wire e2e into check and fix stale tree-less creates `me create` now requires an explicit tree, so the e2e create/search tests that relied on the old default-to-share behavior were failing. Pass an explicit `--tree share`, then add `test:e2e` to the `check` script so the suite is actually exercised. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cli.e2e.test.ts | 14 ++++++++++---- package.json | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index 4f3f81b..8bb806a 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -254,10 +254,12 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(r.stdout).toContain("e2e@example.test"); }); - test("2. create + tree round-trip (default share namespace)", async () => { + test("2. create + tree round-trip (share namespace)", async () => { const created = await meJson<{ id: string; tree?: string }>([ "create", "the quick brown fox jumps over the lazy dog", + "--tree", + "share", ]); expect(created.id).toBeTruthy(); @@ -279,9 +281,11 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( test("4. semantic search ranks a paraphrase near the top", async () => { // Seed a few more memories to make ranking meaningful. - await meJson(["create", "a dog chased a cat across the yard"]); - await meJson(["create", "the stock market fell sharply on Tuesday"]); - await meJson(["create", "photosynthesis converts sunlight into energy"]); + const seed = (text: string) => + meJson(["create", text, "--tree", "share"]); + await seed("a dog chased a cat across the yard"); + await seed("the stock market fell sharply on Tuesday"); + await seed("photosynthesis converts sunlight into energy"); // 4 created so far in `share` (1 from scenario 2 + 3 here). Wait for the // worker to embed them. @@ -312,6 +316,8 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( const created = await meJson<{ id: string }>([ "create", "ephemeral memory to edit", + "--tree", + "share", ]); const updated = await meJson<{ id: string; content: string }>([ "memory", diff --git a/package.json b/package.json index 05657c5..003c1d2 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "scripts": { "build": "./bun run --filter '@memory.build/cli' build", "build:all": "./bun scripts/build-all.ts", - "check": "./bun i --silent && ./bun scripts/bundle-web-assets.ts && ./bun run typecheck && ./bun run lint --write && ./bun run test --only-failures", + "check": "./bun i --silent && ./bun scripts/bundle-web-assets.ts && ./bun run typecheck && ./bun run lint --write && ./bun run test --only-failures && ./bun run test:e2e", "clean": "rm -rf packages/cli/dist dist", "docs": "./bun --filter @memory.build/docs-site dev", "docs:build": "./bun --filter @memory.build/docs-site build", From b9a3485e18155b743d055dff0d5568be204ef6e1 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:46:22 +0200 Subject: [PATCH 127/156] feat(cli): add `me claude init` setup command Adds a first-class `me claude init` subcommand that, for now, backfills existing Claude Code sessions (a default-option import). It's a deliberate seam for further setup steps to be added later, not an alias of `import`. Exports `runAgentImport` from the import command so init reuses the exact auth/option/render path. Covered by an e2e test asserting init backfills a transcript from the default ~/.claude/projects source. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cli.e2e.test.ts | 47 +++++++++++++++++++++++++++++++++ packages/cli/commands/claude.ts | 24 ++++++++++++++++- packages/cli/commands/import.ts | 9 +++++-- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index 8bb806a..584a8aa 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -448,6 +448,53 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( await rm(root, { recursive: true, force: true }); }); + test("8b. `me claude init` backfills existing sessions from the default source", async () => { + // `me claude init` is the one-shot setup command; for now its only step is + // an import with default options (no --source), so it reads the default + // ~/.claude/projects — which, under this run's HOME=tmpHome, resolves to + // tmpHome/.claude/projects. Drop a transcript there and confirm `init` + // backfills it. + const projDir = join(tmpHome, ".claude", "projects", "init-proj"); + await mkdir(projDir, { recursive: true }); + + const sessionId = `init-${rand()}`; + // cwd "/work/init-proj" → no git repo on disk → slug = basename. + const cwd = "/work/init-proj"; + const tree = "share.projects.init_proj.agent_sessions"; + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-03-01T00:00:0${i}.000Z`, + sessionId, + cwd, + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + const lines = [ + mkMsg(0, "user", "init first question"), + mkMsg(1, "assistant", "init first answer"), + mkMsg(2, "user", "init second question"), + mkMsg(3, "assistant", "init second answer"), + ]; + await writeFile( + join(projDir, `${sessionId}.jsonl`), + lines.map((l) => JSON.stringify(l)).join("\n"), + ); + + // Pre-init: nothing captured for this session yet. + expect(await countBySession(sessionId)).toBe(0); + + // `me claude init` (no flags) backfills the existing session. + const init = await me(["claude", "init"]); + expect(init.code, init.stderr).toBe(0); + expect(await countBySession(sessionId)).toBe(4); + expect(await countUnder(tree)).toBe(4); + + await rm(projDir, { recursive: true, force: true }); + }); + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { // A minimal Claude Code session transcript on disk. The importer scans // //*.jsonl; the hook reads the file directly. diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 2f5d2d6..d8d9ab7 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -45,7 +45,7 @@ import { type AgentInstallOptions, runAgentMcpInstall, } from "../mcp/agent-install.ts"; -import { buildAgentImportSubcommand } from "./import.ts"; +import { buildAgentImportSubcommand, runAgentImport } from "./import.ts"; /** GitHub source for `claude plugin marketplace add`. */ const PLUGIN_MARKETPLACE_SOURCE = "timescale/memory-engine"; @@ -426,9 +426,31 @@ function createClaudeHookCommand(): Command { }); } +/** + * me claude init — one-shot setup of Claude Code memory integration. + * + * For now this just backfills your existing Claude Code sessions (the same + * work as `me claude import`). It's a deliberate seam for additional setup + * steps (e.g. installing the plugin, seeding config) to be added here later — + * keep import first so the command stays useful while it grows. + */ +function createClaudeInitCommand(): Command { + return new Command("init") + .description( + "set up Claude Code memory integration (currently: import existing sessions)", + ) + .action(async (_opts, cmd: Command) => { + const globalOpts = cmd.optsWithGlobals(); + // Step 1: backfill existing Claude Code sessions. Run with default import + // options (no flags) — `me claude import` remains the knob-laden variant. + await runAgentImport(claudeImporter, {}, globalOpts); + }); +} + export function createClaudeCommand(): Command { const claude = new Command("claude").description("Claude Code integration"); claude.addCommand(createClaudeInstallCommand()); + claude.addCommand(createClaudeInitCommand()); claude.addCommand(createClaudeHookCommand()); claude.addCommand( buildAgentImportSubcommand( diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index 89a20eb..8b84e47 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -160,8 +160,13 @@ export function buildOptions(opts: Record): { /** * Run one importer end-to-end and render the outcome in the selected format. + * + * Exported so higher-level commands (e.g. `me claude init`) can run an import + * as one step among several, reusing the exact same auth/option/render path as + * the standalone `import` subcommand. `opts` is the parsed import-flag set (pass + * `{}` for defaults); `globalOpts` carries `--server` / output format. */ -async function runAndRender( +export async function runAgentImport( importer: Importer, opts: Record, globalOpts: Record, @@ -320,7 +325,7 @@ export function buildAgentImportSubcommand( addCommonOptions(cmd, includeSidechainsFlag); cmd.action(async (opts, cmdRef) => { const globalOpts = cmdRef.optsWithGlobals(); - await runAndRender(importer, opts, globalOpts); + await runAgentImport(importer, opts, globalOpts); }); return cmd; } From 3ee7a7a408099eebb6c54caf44ce2f12b15ff144 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:53:59 +0200 Subject: [PATCH 128/156] feat(cli): me claude init records project memory location in CLAUDE.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a second step to `me claude init`: after backfilling sessions, it writes (or updates, idempotently via marker comments) a managed block in the project's CLAUDE.md pointing the agent at where this project's memories live in Memory Engine — the `share/projects/` tree, derived with the same SlugRegistry the importer uses, plus how to search them. Targets the git repo root's CLAUDE.md when in a repo, else the cwd's. The e2e test now also asserts the pointer is written and that re-running init leaves exactly one managed block; the `me` helper gained an optional cwd so init runs in an isolated project dir. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cli.e2e.test.ts | 59 ++++++++++++----- packages/cli/commands/claude.ts | 110 +++++++++++++++++++++++++++++--- 2 files changed, 144 insertions(+), 25 deletions(-) diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index 584a8aa..048bdd8 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -19,7 +19,7 @@ process.env.SPACE_SCHEMA_PREFIX = "metest_"; import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { authStore } from "@memory.build/auth"; @@ -162,11 +162,13 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( async function me( args: string[], extraEnv?: Record, + cwd?: string, ): Promise<{ stdout: string; stderr: string; code: number }> { const proc = Bun.spawn([process.execPath, CLI, ...args], { env: cliEnv(extraEnv), stdout: "pipe", stderr: "pipe", + ...(cwd ? { cwd } : {}), }); const [stdout, stderr] = await Promise.all([ new Response(proc.stdout).text(), @@ -448,25 +450,27 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( await rm(root, { recursive: true, force: true }); }); - test("8b. `me claude init` backfills existing sessions from the default source", async () => { - // `me claude init` is the one-shot setup command; for now its only step is - // an import with default options (no --source), so it reads the default - // ~/.claude/projects — which, under this run's HOME=tmpHome, resolves to - // tmpHome/.claude/projects. Drop a transcript there and confirm `init` - // backfills it. - const projDir = join(tmpHome, ".claude", "projects", "init-proj"); - await mkdir(projDir, { recursive: true }); + test("8b. `me claude init` backfills sessions and writes a CLAUDE.md pointer", async () => { + // `me claude init` is the one-shot setup command. Two steps: + // 1. import existing sessions (default --source ~/.claude/projects, + // which under this run's HOME=tmpHome is tmpHome/.claude/projects); + // 2. record the project's memory location in the project's CLAUDE.md + // (the project = init's cwd; not a git repo here → CLAUDE.md lands in + // that dir, slug = its basename). + const transcriptDir = join(tmpHome, ".claude", "projects", "init-proj"); + await mkdir(transcriptDir, { recursive: true }); const sessionId = `init-${rand()}`; - // cwd "/work/init-proj" → no git repo on disk → slug = basename. - const cwd = "/work/init-proj"; + // The transcript's own cwd "/work/init-proj" decides the session tree + // (independent of where `init` is invoked from). + const sessionCwd = "/work/init-proj"; const tree = "share.projects.init_proj.agent_sessions"; const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ type, uuid: `${sessionId}-${type}-${i}`, timestamp: `2026-03-01T00:00:0${i}.000Z`, sessionId, - cwd, + cwd: sessionCwd, message: type === "user" ? { content: text } @@ -479,20 +483,41 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( mkMsg(3, "assistant", "init second answer"), ]; await writeFile( - join(projDir, `${sessionId}.jsonl`), + join(transcriptDir, `${sessionId}.jsonl`), lines.map((l) => JSON.stringify(l)).join("\n"), ); - // Pre-init: nothing captured for this session yet. + // The project we run `init` in — a non-git temp dir with a known basename + // so the derived slug is predictable. CLAUDE.md will be written here. + const projectRoot = await mkdtemp(join(tmpdir(), "me-e2e-initcwd-")); + const projectDir = join(projectRoot, "initcwd"); + await mkdir(projectDir, { recursive: true }); + + // Pre-init: nothing captured, no CLAUDE.md. expect(await countBySession(sessionId)).toBe(0); - // `me claude init` (no flags) backfills the existing session. - const init = await me(["claude", "init"]); + // Run `init` FROM the project dir so its cwd → slug → CLAUDE.md location. + const init = await me(["claude", "init"], undefined, projectDir); expect(init.code, init.stderr).toBe(0); + + // Step 1: the existing session was backfilled. expect(await countBySession(sessionId)).toBe(4); expect(await countUnder(tree)).toBe(4); - await rm(projDir, { recursive: true, force: true }); + // Step 2: CLAUDE.md now points at this project's memories. + const claudeMd = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); + expect(claudeMd).toContain("memory-engine:start"); + expect(claudeMd).toContain("share/projects/initcwd"); + expect(claudeMd).toContain("share/projects/initcwd/agent_sessions"); + + // Re-running is idempotent: still exactly one managed block. + const init2 = await me(["claude", "init"], undefined, projectDir); + expect(init2.code, init2.stderr).toBe(0); + const claudeMd2 = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); + expect(claudeMd2.split("memory-engine:start").length - 1).toBe(1); + + await rm(transcriptDir, { recursive: true, force: true }); + await rm(projectRoot, { recursive: true, force: true }); }); test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index d8d9ab7..6701f6f 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -27,6 +27,7 @@ * Code (no hooks, no slash commands — just the tools). */ import { existsSync } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import * as clack from "@clack/prompts"; import { Command, InvalidArgumentError } from "commander"; @@ -40,7 +41,12 @@ import { import { createMemoryClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { claudeImporter } from "../importers/claude.ts"; -import { importTranscriptFile } from "../importers/index.ts"; +import { + DEFAULT_SESSIONS_NODE_NAME, + DEFAULT_TREE_ROOT, + importTranscriptFile, +} from "../importers/index.ts"; +import { SlugRegistry } from "../importers/slug.ts"; import { type AgentInstallOptions, runAgentMcpInstall, @@ -426,24 +432,112 @@ function createClaudeHookCommand(): Command { }); } +/** Markers delimiting the section `me claude init` manages in a CLAUDE.md. */ +const CLAUDE_MD_START = + ""; +const CLAUDE_MD_END = ""; + +/** Render an ltree path (dot-separated) as the slash form users type into `me search --tree`. */ +function ltreeToSlash(path: string): string { + return path.replaceAll(".", "/"); +} + +/** + * Build the managed CLAUDE.md block that tells an agent where this project's + * memories live in Memory Engine and how to search them. `projectTree` is the + * ltree path (e.g. `share.projects.foo`); `space` is the active space slug, if + * known. + */ +function buildClaudeMdSection(projectTree: string, space?: string): string { + const tree = ltreeToSlash(projectTree); + const sessions = `${tree}/${DEFAULT_SESSIONS_NODE_NAME}`; + const where = space ? `Memory Engine (space \`${space}\`)` : "Memory Engine"; + return [ + CLAUDE_MD_START, + "## Project memories (Memory Engine)", + "", + `Prior context for this project — including captured/imported Claude Code`, + `sessions — is stored in ${where} under the tree:`, + "", + ` ${tree}`, + "", + `- Captured & imported agent sessions: \`${sessions}\``, + `- Search them with the \`me_memory_search\` MCP tool (set \`tree\` to`, + ` \`${tree}\`), or from a shell: \`me search "" --tree ${tree}\`.`, + "", + "Check these before starting work to recall earlier decisions and context.", + CLAUDE_MD_END, + "", + ].join("\n"); +} + +/** + * Upsert the managed Memory Engine section into the project's CLAUDE.md. + * + * Idempotent: if the marker block already exists it is replaced in place; + * otherwise the block is appended (creating the file if absent). Writes to the + * git repo root's CLAUDE.md when in a repo, else the current directory's. + */ +async function writeProjectMemoryPointer(server?: string): Promise { + const cwd = process.cwd(); + const { slug, gitRoot } = await new SlugRegistry().resolve(cwd); + const projectTree = `${DEFAULT_TREE_ROOT}.${slug}`; + const space = resolveCredentials(server).activeSpace; + const section = buildClaudeMdSection(projectTree, space); + + const claudeMdPath = join(gitRoot ?? cwd, "CLAUDE.md"); + let existing = ""; + try { + existing = await readFile(claudeMdPath, "utf8"); + } catch { + existing = ""; // no file yet → create it + } + + let next: string; + const start = existing.indexOf(CLAUDE_MD_START); + if (start !== -1) { + // Replace the existing managed block in place. + const endMarker = existing.indexOf(CLAUDE_MD_END, start); + const end = + endMarker === -1 ? existing.length : endMarker + CLAUDE_MD_END.length; + // Swallow a single trailing newline after the old block so we don't grow + // blank lines on every re-run. + const tail = existing[end] === "\n" ? end + 1 : end; + next = existing.slice(0, start) + section + existing.slice(tail); + } else if (existing.trim().length === 0) { + next = section; + } else { + // Append after the existing content with one blank line of separation. + const sep = existing.endsWith("\n") ? "\n" : "\n\n"; + next = existing + sep + section; + } + + await writeFile(claudeMdPath, next); + clack.log.success(`Recorded project memory location in ${claudeMdPath}`); +} + /** * me claude init — one-shot setup of Claude Code memory integration. * - * For now this just backfills your existing Claude Code sessions (the same - * work as `me claude import`). It's a deliberate seam for additional setup - * steps (e.g. installing the plugin, seeding config) to be added here later — - * keep import first so the command stays useful while it grows. + * Steps (each a deliberate seam for more setup to be added later): + * 1. Backfill existing Claude Code sessions (default-option import — `me + * claude import` remains the knob-laden variant). + * 2. Record where this project's memories live in Memory Engine into the + * project's CLAUDE.md, so the agent knows to consult them. */ function createClaudeInitCommand(): Command { return new Command("init") .description( - "set up Claude Code memory integration (currently: import existing sessions)", + "set up Claude Code memory integration (import existing sessions + note memory location in CLAUDE.md)", ) .action(async (_opts, cmd: Command) => { const globalOpts = cmd.optsWithGlobals(); - // Step 1: backfill existing Claude Code sessions. Run with default import - // options (no flags) — `me claude import` remains the knob-laden variant. + const server = + typeof globalOpts.server === "string" ? globalOpts.server : undefined; + // Step 1: backfill existing Claude Code sessions. await runAgentImport(claudeImporter, {}, globalOpts); + // Step 2: point CLAUDE.md at this project's memories. + await writeProjectMemoryPointer(server); }); } From f2960bc7ebae80b947b7622c38db965364133d56 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 12:57:29 +0200 Subject: [PATCH 129/156] fix(cli): canonical dot tree paths + always-consult instruction in CLAUDE.md The init-written CLAUDE.md block now uses canonical dot-separated ltree paths (share.projects.) rather than slash form, and instructs the agent to always consult project memories first when exploring the codebase or starting a task. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cli.e2e.test.ts | 4 ++-- packages/cli/commands/claude.ts | 20 ++++++++------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index 048bdd8..b811295 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -507,8 +507,8 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( // Step 2: CLAUDE.md now points at this project's memories. const claudeMd = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); expect(claudeMd).toContain("memory-engine:start"); - expect(claudeMd).toContain("share/projects/initcwd"); - expect(claudeMd).toContain("share/projects/initcwd/agent_sessions"); + expect(claudeMd).toContain("share.projects.initcwd"); + expect(claudeMd).toContain("share.projects.initcwd.agent_sessions"); // Re-running is idempotent: still exactly one managed block. const init2 = await me(["claude", "init"], undefined, projectDir); diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 6701f6f..815897e 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -437,20 +437,14 @@ const CLAUDE_MD_START = ""; const CLAUDE_MD_END = ""; -/** Render an ltree path (dot-separated) as the slash form users type into `me search --tree`. */ -function ltreeToSlash(path: string): string { - return path.replaceAll(".", "/"); -} - /** * Build the managed CLAUDE.md block that tells an agent where this project's * memories live in Memory Engine and how to search them. `projectTree` is the - * ltree path (e.g. `share.projects.foo`); `space` is the active space slug, if - * known. + * canonical (dot-separated) ltree path (e.g. `share.projects.foo`); `space` is + * the active space slug, if known. */ function buildClaudeMdSection(projectTree: string, space?: string): string { - const tree = ltreeToSlash(projectTree); - const sessions = `${tree}/${DEFAULT_SESSIONS_NODE_NAME}`; + const sessions = `${projectTree}.${DEFAULT_SESSIONS_NODE_NAME}`; const where = space ? `Memory Engine (space \`${space}\`)` : "Memory Engine"; return [ CLAUDE_MD_START, @@ -459,13 +453,15 @@ function buildClaudeMdSection(projectTree: string, space?: string): string { `Prior context for this project — including captured/imported Claude Code`, `sessions — is stored in ${where} under the tree:`, "", - ` ${tree}`, + ` ${projectTree}`, "", `- Captured & imported agent sessions: \`${sessions}\``, `- Search them with the \`me_memory_search\` MCP tool (set \`tree\` to`, - ` \`${tree}\`), or from a shell: \`me search "" --tree ${tree}\`.`, + ` \`${projectTree}\`), or from a shell: \`me search "" --tree ${projectTree}\`.`, "", - "Check these before starting work to recall earlier decisions and context.", + "Always consult these memories when exploring the codebase or starting a", + "task: search them FIRST to recall earlier decisions and context before", + "digging into the code.", CLAUDE_MD_END, "", ].join("\n"); From 31e56e62cada19ca74b5881a19ce980613e2ef45 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 13:07:14 +0200 Subject: [PATCH 130/156] feat(cli): me claude init step selection (multiselect + --skip flags) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructures `me claude init` around a step registry (INIT_STEPS). In an interactive terminal it presents a multiselect of all steps, each pre-checked, so the user can deselect any; non-interactively it runs every step except those turned off by a generated --skip- flag (--skip-transcript-import, --skip-claude-md). Adding a step is now a single INIT_STEPS entry — it gets both a skip flag and a multiselect row for free. e2e: the `me` helper-driven (piped, non-interactive) tests assert each --skip flag suppresses exactly its step. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cli.e2e.test.ts | 64 ++++++++++++++++ packages/cli/commands/claude.ts | 127 +++++++++++++++++++++++++++----- 2 files changed, 173 insertions(+), 18 deletions(-) diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index b811295..561fde4 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -19,6 +19,7 @@ process.env.SPACE_SCHEMA_PREFIX = "metest_"; import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { existsSync } from "node:fs"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -520,6 +521,69 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( await rm(projectRoot, { recursive: true, force: true }); }); + test("8c. `me claude init` honors --skip-transcript-import / --skip-claude-md", async () => { + // Non-interactive (piped) init runs every step except those turned off by + // a --skip- flag. Verify each flag suppresses exactly its step. + const transcriptDir = join(tmpHome, ".claude", "projects", "skip-proj"); + await mkdir(transcriptDir, { recursive: true }); + const sessionId = `skip-${rand()}`; + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-04-01T00:00:0${i}.000Z`, + sessionId, + cwd: "/work/skip-proj", + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + await writeFile( + join(transcriptDir, `${sessionId}.jsonl`), + [ + mkMsg(0, "user", "skip first question"), + mkMsg(1, "assistant", "skip first answer"), + mkMsg(2, "user", "skip second question"), + mkMsg(3, "assistant", "skip second answer"), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + + const mkProject = async (name: string) => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-skip-")); + const dir = join(root, name); + await mkdir(dir, { recursive: true }); + return { root, dir }; + }; + + // --skip-transcript-import: CLAUDE.md is written, but nothing is imported. + const a = await mkProject("skipimport"); + const r1 = await me( + ["claude", "init", "--skip-transcript-import"], + undefined, + a.dir, + ); + expect(r1.code, r1.stderr).toBe(0); + expect(await countBySession(sessionId)).toBe(0); + expect(existsSync(join(a.dir, "CLAUDE.md"))).toBe(true); + + // --skip-claude-md: the session imports, but no CLAUDE.md is written. + const b = await mkProject("skipclaudemd"); + const r2 = await me( + ["claude", "init", "--skip-claude-md"], + undefined, + b.dir, + ); + expect(r2.code, r2.stderr).toBe(0); + expect(await countBySession(sessionId)).toBe(4); + expect(existsSync(join(b.dir, "CLAUDE.md"))).toBe(false); + + await rm(transcriptDir, { recursive: true, force: true }); + await rm(a.root, { recursive: true, force: true }); + await rm(b.root, { recursive: true, force: true }); + }); + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { // A minimal Claude Code session transcript on disk. The importer scans // //*.jsonl; the hook reads the file directly. diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 815897e..fd772a3 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -51,6 +51,7 @@ import { type AgentInstallOptions, runAgentMcpInstall, } from "../mcp/agent-install.ts"; +import { getOutputFormat } from "../output.ts"; import { buildAgentImportSubcommand, runAgentImport } from "./import.ts"; /** GitHub source for `claude plugin marketplace add`. */ @@ -515,26 +516,116 @@ async function writeProjectMemoryPointer(server?: string): Promise { /** * me claude init — one-shot setup of Claude Code memory integration. * - * Steps (each a deliberate seam for more setup to be added later): - * 1. Backfill existing Claude Code sessions (default-option import — `me - * claude import` remains the knob-laden variant). - * 2. Record where this project's memories live in Memory Engine into the - * project's CLAUDE.md, so the agent knows to consult them. + * Setup is a list of independent steps (see INIT_STEPS). In an interactive + * terminal `init` presents a multiselect of all steps (each pre-checked) so the + * user can deselect any; non-interactively it runs every step except those + * turned off by a `--skip-` flag. To add a step, append one entry to + * INIT_STEPS — it picks up both a `--skip-*` flag and a multiselect row + * automatically. */ +interface InitStepContext { + /** Global CLI opts (carries --server, output format) for the step to use. */ + globalOpts: Record; + /** Resolved server URL, if any. */ + server?: string; +} + +interface InitStep { + /** Stable id — the multiselect value and the basis of the --skip flag. */ + id: string; + /** Commander-parsed key for this step's skip flag (e.g. skipClaudeMd). */ + optionKey: string; + /** The skip flag (e.g. "--skip-claude-md"). */ + skipFlag: string; + /** Help text for the skip flag. */ + skipDescription: string; + /** Multiselect row label. */ + label: string; + /** Multiselect row hint. */ + hint: string; + /** Perform the step. */ + run: (ctx: InitStepContext) => Promise; +} + +const INIT_STEPS: InitStep[] = [ + { + id: "transcript-import", + optionKey: "skipTranscriptImport", + skipFlag: "--skip-transcript-import", + skipDescription: "do not import existing Claude Code sessions", + label: "Import existing Claude Code sessions", + hint: "backfill ~/.claude/projects transcripts into Memory Engine", + run: ({ globalOpts }) => runAgentImport(claudeImporter, {}, globalOpts), + }, + { + id: "claude-md", + optionKey: "skipClaudeMd", + skipFlag: "--skip-claude-md", + skipDescription: + "do not write the memory pointer into the project's CLAUDE.md", + label: "Add a memory pointer to CLAUDE.md", + hint: "tell the agent where this project's memories live", + run: ({ server }) => writeProjectMemoryPointer(server), + }, +]; + function createClaudeInitCommand(): Command { - return new Command("init") - .description( - "set up Claude Code memory integration (import existing sessions + note memory location in CLAUDE.md)", - ) - .action(async (_opts, cmd: Command) => { - const globalOpts = cmd.optsWithGlobals(); - const server = - typeof globalOpts.server === "string" ? globalOpts.server : undefined; - // Step 1: backfill existing Claude Code sessions. - await runAgentImport(claudeImporter, {}, globalOpts); - // Step 2: point CLAUDE.md at this project's memories. - await writeProjectMemoryPointer(server); - }); + const cmd = new Command("init").description( + "set up Claude Code memory integration (interactive step picker; otherwise runs all steps)", + ); + // One --skip- flag per step, so non-interactive runs can opt out. + for (const step of INIT_STEPS) { + cmd.option(step.skipFlag, step.skipDescription); + } + cmd.action(async (opts: Record, cmdRef: Command) => { + const globalOpts = cmdRef.optsWithGlobals(); + const server = + typeof globalOpts.server === "string" ? globalOpts.server : undefined; + + // Baseline = every step not explicitly turned off via its --skip-* flag. + const baseline = INIT_STEPS.filter((s) => opts[s.optionKey] !== true); + + // Interactive (a TTY with text output): present a multiselect pre-checked + // with the baseline so the user can deselect steps. Otherwise run the + // baseline as-is. + const interactive = + getOutputFormat(globalOpts) === "text" && + Boolean(process.stdin.isTTY) && + Boolean(process.stdout.isTTY); + + let selectedIds: string[]; + if (interactive) { + const picked = await clack.multiselect({ + message: "Select setup steps to run", + options: INIT_STEPS.map((s) => ({ + value: s.id, + label: s.label, + hint: s.hint, + })), + initialValues: baseline.map((s) => s.id), + required: false, + }); + if (clack.isCancel(picked)) { + clack.cancel("Cancelled."); + process.exit(0); + } + selectedIds = picked; + } else { + selectedIds = baseline.map((s) => s.id); + } + + const selected = INIT_STEPS.filter((s) => selectedIds.includes(s.id)); + if (selected.length === 0) { + clack.log.info("No setup steps selected — nothing to do."); + return; + } + + const ctx: InitStepContext = { globalOpts, server }; + for (const step of selected) { + await step.run(ctx); + } + }); + return cmd; } export function createClaudeCommand(): Command { From 4320dd70754f189f43735624fcb0de8ec7e16cbf Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 13:52:43 +0200 Subject: [PATCH 131/156] fix(cli): clearer instructions in me claude init step multiselect Spell out in the multiselect prompt that all steps are selected by default and how to toggle (arrows move, space toggles, enter confirms), since the toggle interaction isn't self-evident. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/claude.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index fd772a3..d1d6834 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -596,7 +596,8 @@ function createClaudeInitCommand(): Command { let selectedIds: string[]; if (interactive) { const picked = await clack.multiselect({ - message: "Select setup steps to run", + message: + "Setup steps to run (all selected by default) — ↑/↓ move, space to toggle a step off/on, enter to confirm:", options: INIT_STEPS.map((s) => ({ value: s.id, label: s.label, From bc863fe3d0a131ab7777f9446e60ee5932a256e3 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 14:01:18 +0200 Subject: [PATCH 132/156] style(cli): dim the instruction hint in me claude init multiselect Render the parenthetical guidance on the multiselect prompt line in dim ANSI so the main "Setup steps to run" text stands out, and drop the per-option hint rows (and the now-unused InitStep.hint field) for plain, uncluttered step labels. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/commands/claude.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index d1d6834..1e21916 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -438,6 +438,11 @@ const CLAUDE_MD_START = ""; const CLAUDE_MD_END = ""; +/** Dim (secondary text) ANSI, for de-emphasizing hint copy. `\x1b[22m` resets + * only the dim attribute so surrounding clack styling is left intact. */ +const DIM = "\x1b[2m"; +const DIM_OFF = "\x1b[22m"; + /** * Build the managed CLAUDE.md block that tells an agent where this project's * memories live in Memory Engine and how to search them. `projectTree` is the @@ -541,8 +546,6 @@ interface InitStep { skipDescription: string; /** Multiselect row label. */ label: string; - /** Multiselect row hint. */ - hint: string; /** Perform the step. */ run: (ctx: InitStepContext) => Promise; } @@ -554,7 +557,6 @@ const INIT_STEPS: InitStep[] = [ skipFlag: "--skip-transcript-import", skipDescription: "do not import existing Claude Code sessions", label: "Import existing Claude Code sessions", - hint: "backfill ~/.claude/projects transcripts into Memory Engine", run: ({ globalOpts }) => runAgentImport(claudeImporter, {}, globalOpts), }, { @@ -564,7 +566,6 @@ const INIT_STEPS: InitStep[] = [ skipDescription: "do not write the memory pointer into the project's CLAUDE.md", label: "Add a memory pointer to CLAUDE.md", - hint: "tell the agent where this project's memories live", run: ({ server }) => writeProjectMemoryPointer(server), }, ]; @@ -596,12 +597,10 @@ function createClaudeInitCommand(): Command { let selectedIds: string[]; if (interactive) { const picked = await clack.multiselect({ - message: - "Setup steps to run (all selected by default) — ↑/↓ move, space to toggle a step off/on, enter to confirm:", + message: `Setup steps to run ${DIM}(all selected by default — ↑/↓ move, space to toggle off/on, enter to confirm)${DIM_OFF}`, options: INIT_STEPS.map((s) => ({ value: s.id, label: s.label, - hint: s.hint, })), initialValues: baseline.map((s) => s.id), required: false, From 0d136977fb64ed6ab190a79bd7b2c57e3a64e86e Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 16:36:22 +0200 Subject: [PATCH 133/156] fix(server): batch create skips duplicate explicit ids instead of erroring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_memory now does `on conflict (id) do nothing` and returns null for a duplicate explicit id; memory.batchCreate returns inserted ids only and memory.create maps the null to a clean CONFLICT error (previously an unmapped unique_violation surfaced as "Internal error" and failed the whole batch). This makes the long-documented import contract real: chunk.ts, the MCP import tool's idempotentHint, and docs/cli/me-memory.md all promised ON CONFLICT skip semantics the SQL never had. The session importers never tripped it (they pre-diff existing state via search), but any path that re-submits explicit ids — `me memory import` re-imports, or a session whose >1000 already-imported messages overflow the capped dedup lookup — failed its whole chunk with "Internal error". Co-Authored-By: Claude Fable 5 --- .../space/migrate/idempotent/001_memory.sql | 7 +++- .../space/migrate/migrate.integration.test.ts | 23 +++++++++++ packages/engine/space/db.integration.test.ts | 41 ++++++++++++++++--- packages/engine/space/db.ts | 10 ++++- packages/server/rpc/memory/memory.ts | 31 +++++++++----- 5 files changed, 93 insertions(+), 19 deletions(-) diff --git a/packages/database/space/migrate/idempotent/001_memory.sql b/packages/database/space/migrate/idempotent/001_memory.sql index c4048d8..35b133f 100644 --- a/packages/database/space/migrate/idempotent/001_memory.sql +++ b/packages/database/space/migrate/idempotent/001_memory.sql @@ -99,6 +99,10 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ------------------------------------------------------------------------------- -- create memory +-- +-- Returns the new memory's id, or null when an explicit _id already exists +-- (on conflict do nothing). The null lets importers with deterministic ids +-- re-submit safely — the caller classifies a missing id as "skipped". ------------------------------------------------------------------------------- create or replace function {{schema}}.create_memory ( _tree_access jsonb @@ -130,7 +134,8 @@ begin , _temporal , _content ) - returning id into strict _id + on conflict (id) do nothing + returning id into _id ; return _id; end; diff --git a/packages/database/space/migrate/migrate.integration.test.ts b/packages/database/space/migrate/migrate.integration.test.ts index d8da816..cc58bf8 100644 --- a/packages/database/space/migrate/migrate.integration.test.ts +++ b/packages/database/space/migrate/migrate.integration.test.ts @@ -286,6 +286,29 @@ describe("provisioned schema is functional", () => { expect(updated?.updated_at).not.toBeNull(); }); + test("create_memory skips a duplicate explicit id (returns null)", async () => { + // Deterministic-id importers re-submit existing ids; the second create + // must be a no-op that returns null, leaving the original row intact. + const owner = `'[{"tree_path": "", "access": 3}]'::jsonb`; + const id = "01941000-0000-7000-8000-000000000001"; + const [first] = await sql.unsafe( + `select ${canonical.schema}.create_memory( + ${owner}, 'a.dup'::ltree, 'original', '${id}'::uuid) as id`, + ); + expect(first?.id).toBe(id); + + const [second] = await sql.unsafe( + `select ${canonical.schema}.create_memory( + ${owner}, 'a.dup'::ltree, 'replacement', '${id}'::uuid) as id`, + ); + expect(second?.id).toBeNull(); + + const [row] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("original"); + }); + test("enforces the meta-is-object constraint", async () => { await expectReject(() => sql.unsafe( diff --git a/packages/engine/space/db.integration.test.ts b/packages/engine/space/db.integration.test.ts index 2600ec4..faae91c 100644 --- a/packages/engine/space/db.integration.test.ts +++ b/packages/engine/space/db.integration.test.ts @@ -51,8 +51,18 @@ async function setEmbedding(id: string, vec: number[]): Promise { ); } +/** createMemory asserting the insert happened (no duplicate-id skip). */ +async function mustCreate( + access: TreeAccess, + params: Parameters[1], +): Promise { + const id = await db.createMemory(access, params); + if (id === null) throw new Error("unexpected duplicate-id skip"); + return id; +} + test("createMemory + getMemory round-trips", async () => { - const id = await db.createMemory(FULL, { + const id = await mustCreate(FULL, { tree: "work.note", content: "hello world", meta: { kind: "note" }, @@ -65,6 +75,25 @@ test("createMemory + getMemory round-trips", async () => { expect(m?.hasEmbedding).toBe(false); }); +test("createMemory returns null for a duplicate explicit id", async () => { + const id = "01900000-0000-7000-8000-0000000000d0"; + const first = await db.createMemory(FULL, { + id, + tree: "work.dup", + content: "original", + }); + expect(first).toBe(id); + + // Re-submitting the same id is a no-op skip, not an error. + const second = await db.createMemory(FULL, { + id, + tree: "work.dup", + content: "replacement", + }); + expect(second).toBeNull(); + expect((await db.getMemory(FULL, id))?.content).toBe("original"); +}); + test("access is enforced by the tree_access argument", async () => { // create requires write (>=2): read-only access is rejected await expect( @@ -72,7 +101,7 @@ test("access is enforced by the tree_access argument", async () => { ).rejects.toThrow(); // a memory is invisible to a tree_access set that doesn't cover its path - const id = await db.createMemory(FULL, { + const id = await mustCreate(FULL, { tree: "work.secret", content: "shh", }); @@ -81,7 +110,7 @@ test("access is enforced by the tree_access argument", async () => { }); test("patchMemory updates fields; deleteMemory removes", async () => { - const id = await db.createMemory(FULL, { + const id = await mustCreate(FULL, { tree: "work.p", content: "before", }); @@ -137,11 +166,11 @@ test("unranked (filter-only) search orders by id, newest-first by default", asyn }); test("vector search ranks by embedding similarity", async () => { - const near = await db.createMemory(FULL, { + const near = await mustCreate(FULL, { tree: "work.v1", content: "near", }); - const far = await db.createMemory(FULL, { tree: "work.v2", content: "far" }); + const far = await mustCreate(FULL, { tree: "work.v2", content: "far" }); await setEmbedding(near, [1, 0, 0, 0]); await setEmbedding(far, [0, 1, 0, 0]); @@ -150,7 +179,7 @@ test("vector search ranks by embedding similarity", async () => { }); test("hybridSearch fuses bm25 + vector", async () => { - const id = await db.createMemory(FULL, { + const id = await mustCreate(FULL, { tree: "work.h", content: "hybrid pineapple", }); diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts index 53cc6a9..2dbd157 100644 --- a/packages/engine/space/db.ts +++ b/packages/engine/space/db.ts @@ -19,10 +19,15 @@ import type { * No table queries in TS; no RLS (access is the jsonb argument). */ export interface SpaceStore { + /** + * Insert one memory, returning its id — or null when an explicit + * `params.id` already exists (`on conflict do nothing`), so deterministic-id + * importers can re-submit idempotently. + */ createMemory( treeAccess: TreeAccess, params: CreateMemoryParams, - ): Promise; + ): Promise; getMemory(treeAccess: TreeAccess, id: string): Promise; patchMemory( treeAccess: TreeAccess, @@ -122,7 +127,8 @@ export function spaceStore(sql: Sql, schema: string): SpaceStore { ${p.temporal ?? null}::tstzrange ) as id`; if (!row) throw new Error("create_memory returned no row"); - return row.id as string; + // Null id = the explicit id already exists (on conflict do nothing). + return (row.id as string | null) ?? null; }, async getMemory(treeAccess, id) { diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 15dc7c0..6256373 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -186,6 +186,11 @@ async function memoryCreate( temporal: formatTemporal(params.temporal), }), ); + if (id === null) { + // The store skips an explicit id that already exists (on conflict do + // nothing). For a single create that's a caller error, not a skip. + throw new AppError("CONFLICT", `Memory already exists: ${params.id}`); + } const memory = await store.getMemory(treeAccess, id); if (!memory) { throw new AppError("INTERNAL_ERROR", "Created memory could not be read"); @@ -193,7 +198,14 @@ async function memoryCreate( return toMemoryResponse(memory, ctx); } -/** memory.batchCreate — atomic across the batch. */ +/** + * memory.batchCreate — atomic across the batch. + * + * Returns the inserted ids only: a memory whose explicit id already exists + * is silently skipped (`on conflict do nothing` in create_memory), so + * deterministic-id importers can re-submit and classify the missing ids as + * already imported (see `computeSkippedIds` in the CLI). + */ async function memoryBatchCreate( params: MemoryBatchCreateParams, context: HandlerContext, @@ -206,15 +218,14 @@ async function memoryBatchCreate( store.withTransaction(async (tx) => { const out: string[] = []; for (const m of params.memories) { - out.push( - await tx.createMemory(treeAccess, { - id: m.id ?? undefined, - content: m.content, - meta: m.meta ?? undefined, - tree: inputTreePath(ctx, m.tree), - temporal: formatTemporal(m.temporal), - }), - ); + const id = await tx.createMemory(treeAccess, { + id: m.id ?? undefined, + content: m.content, + meta: m.meta ?? undefined, + tree: inputTreePath(ctx, m.tree), + temporal: formatTemporal(m.temporal), + }); + if (id !== null) out.push(id); } return out; }), From fc027722ca24224a4a69645c0c6c4dd04bd4cd2a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 16:36:46 +0200 Subject: [PATCH 134/156] feat(cli): `me import` source group + git history importer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure imports under one umbrella group — `me import memories|claude|codex|opencode|git` — so each new source is one subcommand instead of a new top-level command group. The old spellings stay registered as aliases (`me memory import`, `me claude|codex|opencode import`); the bare `me import ` alias is gone (the group owns the top-level name, and the group help points old muscle memory at `me import memories`). New `me import git [repo]`: one memory per commit — message plus a capped changed-file list — under `..git_history`, with the commit date as temporal and a deterministic UUIDv7 keyed by (tree, sha) so re-imports are server-side no-op skips. Re-runs are incremental: one search finds the newest imported sha and, when it is an ancestor of the target rev, only `..` is walked (force-pushes fall back to the full walk, which the deterministic ids make safe). The walk is one streamed `git log --numstat` with NUL-delimited fields, parsed incrementally in constant memory. Body-less merge boilerplate is skipped; merges with a body (PR merges) are kept. `me claude init` gains an "Import git commit history" step (--skip-git-import; soft-skips outside a git repo), and the managed CLAUDE.md block now names the git_history node next to agent_sessions. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 4 +- docs/cli/agent-session-imports.md | 10 +- docs/cli/me-claude.md | 23 +- docs/cli/me-codex.md | 2 +- docs/cli/me-import.md | 115 ++++++ docs/cli/me-memory.md | 2 +- docs/cli/me-opencode.md | 2 +- docs/formats.md | 4 +- e2e/cli.e2e.test.ts | 200 +++++++++- packages/cli/claude/capture.ts | 4 +- packages/cli/commands/claude.ts | 26 +- packages/cli/commands/codex.ts | 10 +- packages/cli/commands/import-git.test.ts | 62 +++ packages/cli/commands/import-git.ts | 356 +++++++++++++++++ packages/cli/commands/import-group.ts | 41 ++ packages/cli/commands/import.ts | 64 ++- packages/cli/commands/memory-import.test.ts | 2 +- packages/cli/commands/memory-import.ts | 4 +- packages/cli/commands/memory.ts | 7 +- packages/cli/commands/opencode.ts | 10 +- packages/cli/importers/git.test.ts | 287 ++++++++++++++ packages/cli/importers/git.ts | 372 ++++++++++++++++++ .../cli/importers/import-transcript.test.ts | 8 +- packages/cli/importers/index.ts | 9 +- packages/cli/importers/uuid.test.ts | 23 +- packages/cli/importers/uuid.ts | 44 ++- packages/cli/index.ts | 4 + packages/docs-site/lib/nav.ts | 1 + scripts/integration-test.ts | 12 + 29 files changed, 1620 insertions(+), 88 deletions(-) create mode 100644 docs/cli/me-import.md create mode 100644 packages/cli/commands/import-git.test.ts create mode 100644 packages/cli/commands/import-git.ts create mode 100644 packages/cli/commands/import-group.ts create mode 100644 packages/cli/importers/git.test.ts create mode 100644 packages/cli/importers/git.ts diff --git a/CLAUDE.md b/CLAUDE.md index 58f3f6e..38a094b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,14 +29,14 @@ Read the relevant docs before starting work on a subsystem. - **Memory table** (per space): `content`, `meta` (JSONB), `tree` (ltree), `temporal` (tstzrange), `embedding` (halfvec(1536)). - **Search**: hybrid BM25 + semantic via Reciprocal Rank Fusion, computed in SQL functions. - **Access**: no RLS. `core.build_tree_access(principalId, spaceId)` produces a `_tree_access` jsonb (rows of `tree_path` + `access`) passed into the space SQL functions (`search_memory`, `get_memory`, …). Three additive levels: **1 = read, 2 = write, 3 = owner**; `owner@root` (the empty ltree path) owns the whole space, and an owner grant at any path delegates access-management within that subtree. Two axes: **structural** authority (`principal_space.admin` — roster mutations, groups, invitations) vs **data** authority (owner@path); an admin may also grant data and can self-grant `owner@root`. The auth gate is a non-empty `build_tree_access` (every member holds ≥1 grant). -- **Tree conventions**: two reserved roots — per-member `home.` (`~` is input sugar for it; a joining **user** is granted `owner@home`) and the shared `share`. A space **creator** gets `admin` + `owner@home` + `owner@share`, **not** `owner@root` — so it sees `share` and its own `~` but not other members' homes (as an admin it can self-grant `owner@root`). `memory.create`/`batchCreate` **require** an explicit `tree` (callers choose `share` vs `~` deliberately); only the file importers (`me memory import`, the `me_memory_import` MCP tool) default a tree-less record to `share` (`SHARE_NAMESPACE`, canonically defined in `@memory.build/protocol` and re-exported by `@memory.build/database`). +- **Tree conventions**: two reserved roots — per-member `home.` (`~` is input sugar for it; a joining **user** is granted `owner@home`) and the shared `share`. A space **creator** gets `admin` + `owner@home` + `owner@share`, **not** `owner@root` — so it sees `share` and its own `~` but not other members' homes (as an admin it can self-grant `owner@root`). `memory.create`/`batchCreate` **require** an explicit `tree` (callers choose `share` vs `~` deliberately); only the file importers (`me import memories`, the `me_memory_import` MCP tool) default a tree-less record to `share` (`SHARE_NAMESPACE`, canonically defined in `@memory.build/protocol` and re-exported by `@memory.build/database`). - **API**: JSON-RPC 2.0 over HTTP, two endpoints: - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `invite.*`). - `/api/v1/user/rpc` — session only (an api key never authenticates here; agents can't manage agents). `whoami`, `agent.*`, `apiKey.*`, `space.*`. - Plus REST OAuth device-flow endpoints under `/api/v1/auth/*`. - **Auth**: humans use a **session token** (OAuth device flow, GitHub/Google); agents use an **api key** (`me..`). Api keys are **global** per-principal credentials, not space-bound: the same key works in any space the agent has been admitted to (the space comes from `X-Me-Space`, gated by `build_tree_access`). Session + api-key secrets are sha256 (compared by equality in SQL), not argon2. - **Embedding**: Vercel AI SDK; OpenAI `text-embedding-3-small` (1536-dim) in production; Ollama supported for local dev. -- **CLI**: `me` binary — `login`, `logout`, `whoami`, `space`, `group`, `access`, `agent`, `apikey`, `memory` (+ top-level aliases like `me search`, `me create`), `mcp`, `claude`/`codex`/`gemini`/`opencode`, `serve`, `pack`. +- **CLI**: `me` binary — `login`, `logout`, `whoami`, `space`, `group`, `access`, `agent`, `apikey`, `memory` (+ top-level aliases like `me search`, `me create` — except `import`), `import` (the source group: `memories`/`claude`/`codex`/`opencode`/`git`; `me memory import` and `me import` remain as aliases), `mcp`, `claude`/`codex`/`gemini`/`opencode`, `serve`, `pack`. ## Principals, members, spaces (terminology) diff --git a/docs/cli/agent-session-imports.md b/docs/cli/agent-session-imports.md index 793af8a..1be2961 100644 --- a/docs/cli/agent-session-imports.md +++ b/docs/cli/agent-session-imports.md @@ -1,16 +1,16 @@ # Agent session imports -Shared reference for the per-agent `import` subcommands: +Shared reference for the agent-session import subcommands: -- [`me claude import`](me-claude.md#me-claude-import) -- [`me codex import`](me-codex.md#me-codex-import) -- [`me opencode import`](me-opencode.md#me-opencode-import) +- `me import claude` ([`me claude import`](me-claude.md#me-claude-import) is its alias) +- `me import codex` ([`me codex import`](me-codex.md#me-codex-import) is its alias) +- `me import opencode` ([`me opencode import`](me-opencode.md#me-opencode-import) is its alias) Each source-native message becomes one memory. Re-running the same command only inserts newly-seen messages (deterministic UUIDs make re-imports idempotent). ## Shared options -All three subcommands accept the same flags (with one extra flag on `me claude import`). +All three subcommands accept the same flags (with one extra flag on the Claude importer). | Option | Description | |--------|-------------| diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index 80d7630..2ce630c 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -5,6 +5,7 @@ Claude Code integration commands. ## Commands - [me claude install](#me-claude-install) -- install the Memory Engine plugin for Claude Code (full plugin by default, `--mcp-only` for just the MCP server) +- [me claude init](#me-claude-init) -- one-shot setup: backfill sessions, import git history, record the project's memory location in CLAUDE.md - [me claude hook](#me-claude-hook) -- invoked by the Claude Code plugin to capture events as memories - [me claude import](#me-claude-import) -- import Claude Code sessions from `~/.claude/projects` @@ -52,6 +53,26 @@ For manual MCP client configuration, see [MCP Integration](../mcp-integration.md --- +## me claude init + +One-shot setup of Claude Code memory integration for the current project. + +``` +me claude init [options] +``` + +Setup is a list of independent steps. In an interactive terminal `init` presents a multiselect of all steps (each pre-checked) so you can deselect any; non-interactively it runs every step except those turned off by a `--skip-` flag. + +| Step | Skip flag | What it does | +|------|-----------|--------------| +| Import existing Claude Code sessions | `--skip-transcript-import` | Backfills sessions from `~/.claude/projects` — the same import as [`me import claude`](me-import.md#me-import-claude--codex--opencode). | +| Import git commit history | `--skip-git-import` | Imports the repo's full commit history — the same import as [`me import git`](me-import.md#me-import-git). Skipped automatically when the current directory is not inside a git repo. | +| Add a memory pointer to CLAUDE.md | `--skip-claude-md` | Upserts a managed block into the project's CLAUDE.md naming the project tree (`share.projects.`), its `agent_sessions` and `git_history` nodes, and how to search them. Idempotent — re-runs replace the block in place. | + +Re-running `init` is safe: both imports are incremental/idempotent and the CLAUDE.md block is replaced, not duplicated. + +--- + ## me claude hook Invoked by the Claude Code plugin on `Stop` (each turn) and `SessionEnd`. Reads the `transcript_path` from the event JSON on stdin, resolves config from `CLAUDE_PLUGIN_OPTION_*` env vars (falling back to your `me login` session), and imports the session transcript — the same parse + write as [`me … import`](agent-session-imports.md), incremental so each call only writes messages new since the last. @@ -83,7 +104,7 @@ Best-effort: logs failures to stderr but always exits 0 so that a hook failure n ## me claude import -Import Claude Code sessions from `~/.claude/projects//.jsonl`. +Import Claude Code sessions from `~/.claude/projects//.jsonl`. This is an alias of [`me import claude`](me-import.md#me-import-claude--codex--opencode). ``` me claude import [options] diff --git a/docs/cli/me-codex.md b/docs/cli/me-codex.md index 24ad505..ea6eff4 100644 --- a/docs/cli/me-codex.md +++ b/docs/cli/me-codex.md @@ -31,7 +31,7 @@ For manual MCP client configuration, see [MCP Integration](../mcp-integration.md ## me codex import -Import Codex sessions from `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl` and `~/.codex/archived_sessions/*.jsonl`. +Import Codex sessions from `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl` and `~/.codex/archived_sessions/*.jsonl`. This is an alias of [`me import codex`](me-import.md#me-import-claude--codex--opencode). ``` me codex import [options] diff --git a/docs/cli/me-import.md b/docs/cli/me-import.md new file mode 100644 index 0000000..932aca3 --- /dev/null +++ b/docs/cli/me-import.md @@ -0,0 +1,115 @@ +# me import + +Get data into Memory Engine — one subcommand per source. + +## Commands + +- [me import memories](#me-import-memories) -- import memory records from files or stdin (md/yaml/json/ndjson) +- [me import claude](#me-import-claude--codex--opencode) -- import Claude Code sessions +- [me import codex](#me-import-claude--codex--opencode) -- import Codex sessions +- [me import opencode](#me-import-claude--codex--opencode) -- import OpenCode sessions +- [me import git](#me-import-git) -- import a repo's git commit history + +There is no bare default: `me import ` does not parse — use `me import memories `. + +--- + +## me import memories + +Import memory records from files or stdin. `me memory import` is an alias of this command. + +``` +me import memories [files...] [options] +``` + +See [me memory import](me-memory.md#me-memory-import) for the full option reference, format detection, skip semantics, and chunking behavior, and [File Formats](../formats.md) for the record schemas. + +--- + +## me import claude / codex / opencode + +Import agent sessions from each tool's native storage. The per-agent spellings (`me claude import`, `me codex import`, `me opencode import`) are aliases of these commands. + +``` +me import claude [options] +me import codex [options] +me import opencode [options] +``` + +See [agent session imports](agent-session-imports.md) for the shared option reference, tree layout, idempotency rules, content shape, and metadata schema. + +--- + +## me import git + +Import a repo's git commit history as memories — one memory per commit, holding the commit message plus a capped changed-file list. Commit intent ("why did we do X") and touched paths become searchable agent context. + +``` +me import git [repo] [options] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `repo` | no | Path inside the repo to import. Default: the current directory. | + +| Option | Description | +|--------|-------------| +| `--branch ` | Branch, tag, or rev to walk. Default: `HEAD`. | +| `--since ` | Only commits at/after this date (any format git accepts). | +| `--until ` | Only commits at/before this date. | +| `--max-count ` | Import at most this many recent commits. | +| `--full` | Walk the full history (skip the incremental high-water lookup). | +| `--no-merges` | Drop all merge commits. | +| `--no-file-list` | Omit the changed-file list from commit memories. | +| `--tree-root ` | Tree root under which `.git_history` is placed. Default: `share.projects`. | +| `--dry-run` | Parse and report what would be imported without writing. | +| `-v, --verbose` | Per-commit progress output. | + +### Tree layout + +Commits are stored under: + +``` +..git_history +``` + +The project slug is derived exactly as for [agent session imports](agent-session-imports.md#tree-layout) (git remote repo name, else repo root directory name), so a project's commit history sits next to its `agent_sessions` node — e.g. `share.projects.memory_engine.git_history`. + +### Content shape + +Each memory's content is the commit subject, the body (truncated past 64 KiB), and a `Files:` block listing up to 50 changed paths with `(+added -deleted)` line counts (`(binary)` for binary files). `--no-file-list` omits the block. + +Merge commits with no message body (`Merge branch 'x'` boilerplate) are skipped by default; merges that carry a body — GitHub PR merge commits put the PR title there — are imported. `--no-merges` drops all merges. + +### Idempotency and incremental re-runs + +Each commit gets a deterministic UUIDv7 keyed by `(tree, sha)` with the commit date as its timestamp half. Re-imports are server-side no-ops: an already-imported commit is skipped, never duplicated. + +Re-runs are also incremental: the newest already-imported commit is looked up server-side, and when it is an ancestor of the target rev only `..` is walked. After a force-push (or when importing a different branch) the walk falls back to the full log — still safe, because the deterministic ids dedupe the overlap. Explicit bounds (`--since`, `--until`, `--max-count`, `--full`) always walk exactly what they say. + +### Metadata + +| Key | Description | +|-----|-------------| +| `type` | Always `"git_commit"`. | +| `sha` | Full 40-hex commit sha. | +| `source_git_repo` | Git remote URL (when the repo has one). | +| `source_project_slug` | ltree-safe project label (same as the tree subnode). | +| `author_name` / `author_email` | Commit author. | +| `author_date` / `commit_date` | ISO 8601 author and committer dates. | +| `files_changed` / `insertions` / `deletions` | Change stats (binary files excluded from line counts). | +| `is_merge` | `true` on merge commits (absent otherwise). | +| `imported_at` | ISO 8601 timestamp of this import run. | +| `importer_version` | Version tag of the importer schema. | + +Temporal is a point-in-time at the commit date. + +### Example + +Backfill this repo's history, then keep it current with cheap re-runs: + +```bash +me import git --dry-run -v # preview +me import git # full backfill (first run) +me import git # later: walks only commits since the last import +``` diff --git a/docs/cli/me-memory.md b/docs/cli/me-memory.md index 88100bd..75ba084 100644 --- a/docs/cli/me-memory.md +++ b/docs/cli/me-memory.md @@ -213,7 +213,7 @@ Moves all memories under the source prefix to the destination, preserving subtre ## me memory import -Import memories from files or stdin. +Import memories from files or stdin. This is an alias of [`me import memories`](me-import.md#me-import-memories) (unlike the other memory subcommands, `import` has no bare top-level alias — the top-level `me import` is the [import group](me-import.md)). ``` me memory import [files...] [options] diff --git a/docs/cli/me-opencode.md b/docs/cli/me-opencode.md index b9fb2df..2d283a5 100644 --- a/docs/cli/me-opencode.md +++ b/docs/cli/me-opencode.md @@ -31,7 +31,7 @@ For manual MCP client configuration, see [MCP Integration](../mcp-integration.md ## me opencode import -Import OpenCode sessions from `~/.local/share/opencode/storage/`. +Import OpenCode sessions from `~/.local/share/opencode/storage/`. This is an alias of [`me import opencode`](me-import.md#me-import-claude--codex--opencode). ``` me opencode import [options] diff --git a/docs/formats.md b/docs/formats.md index afa6454..0f29ad5 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -1,6 +1,6 @@ # File Formats -Import and export use the same memory structure across all formats. This page is the canonical reference for the JSON, YAML, Markdown, and NDJSON schemas used by both the CLI (`me memory import` / `me memory export`) and MCP tools (`me_memory_import` / `me_memory_export`). +Import and export use the same memory structure across all formats. This page is the canonical reference for the JSON, YAML, Markdown, and NDJSON schemas used by both the CLI (`me import memories` — alias `me memory import` — / `me memory export`) and MCP tools (`me_memory_import` / `me_memory_export`). ## Memory fields @@ -11,7 +11,7 @@ Every memory has one required field (`content`) and four optional fields: | `id` | `string` | no | UUIDv7. Enables idempotent imports -- re-importing the same ID won't create a duplicate. Must match `^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`. | | `content` | `string` | **yes** | The memory text. Must be non-empty. | | `meta` | `object` | no | Arbitrary key-value metadata. Any valid JSON object. | -| `tree` | `string` | no | Hierarchical path using dot-separated labels (e.g. `share.work.projects.api`). Labels match `[A-Za-z0-9_-]`; `/` is also accepted as a separator and a leading `~` expands to your private home. When omitted, the file importers (`me memory import`, `me_memory_import`) default the record to the shared root `share`. | +| `tree` | `string` | no | Hierarchical path using dot-separated labels (e.g. `share.work.projects.api`). Labels match `[A-Za-z0-9_-]`; `/` is also accepted as a separator and a leading `~` expands to your private home. When omitted, the file importers (`me import memories`, `me_memory_import`) default the record to the shared root `share`. | | `temporal` | varies | no | Time range for the memory. Accepted shapes depend on format -- see below. | ### Temporal input shapes diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index 561fde4..b01bda9 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -438,8 +438,9 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(await countBySession(newSession)).toBe(4); expect(await countBySession(oldSession)).toBe(0); - // 3. Run `me claude import`. - const imp = await me(["claude", "import", "--source", root]); + // 3. Run the import (canonical spelling; test 9 covers the + // `me claude import` alias). + const imp = await me(["import", "claude", "--source", root]); expect(imp.code, imp.stderr).toBe(0); // 4. The pre-install work is now backfilled, and the hook's live capture @@ -510,6 +511,7 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(claudeMd).toContain("memory-engine:start"); expect(claudeMd).toContain("share.projects.initcwd"); expect(claudeMd).toContain("share.projects.initcwd.agent_sessions"); + expect(claudeMd).toContain("share.projects.initcwd.git_history"); // Re-running is idempotent: still exactly one managed block. const init2 = await me(["claude", "init"], undefined, projectDir); @@ -584,6 +586,174 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( await rm(b.root, { recursive: true, force: true }); }); + // Run git in `dir`, isolated from the developer's git config (gpg + // signing, hooks, templates), with deterministic commit dates. + async function git( + dir: string, + args: string[], + dateIso?: string, + ): Promise { + const proc = Bun.spawn( + [ + "git", + "-C", + dir, + "-c", + "user.name=E2E", + "-c", + "user.email=e2e@example.test", + "-c", + "commit.gpgsign=false", + ...args, + ], + { + env: { + ...process.env, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + ...(dateIso + ? { GIT_AUTHOR_DATE: dateIso, GIT_COMMITTER_DATE: dateIso } + : {}), + }, + stdout: "pipe", + stderr: "pipe", + }, + ); + const code = await proc.exited; + if (code !== 0) { + const stderr = await new Response(proc.stderr).text(); + throw new Error(`git ${args.join(" ")} failed: ${stderr}`); + } + } + + test("8d. `me import git` imports commit history, idempotently and incrementally", async () => { + // A real repo with a known-basename root so the slug (no remote → + // basename) and therefore the tree are predictable. + const root = await mkdtemp(join(tmpdir(), "me-e2e-git-")); + const name = `gitproj${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + const commitFile = async (file: string, msg: string, dateIso: string) => { + await writeFile(join(repo, file), `${msg}\n`); + await git(repo, ["add", "."], dateIso); + await git(repo, ["commit", "-q", "-m", msg], dateIso); + }; + await commitFile("a.txt", "feat: add a", "2026-05-01T10:00:00Z"); + await commitFile("b.txt", "fix: adjust b", "2026-05-02T10:00:00Z"); + await commitFile("c.txt", "docs: describe c", "2026-05-03T10:00:00Z"); + + // 1. First import: all three commits land under the project tree. + const first = await meJson<{ + inserted: number; + commitsWalked: number; + tree: string; + }>(["import", "git", repo]); + expect(first.tree).toBe(tree); + expect(first.commitsWalked).toBe(3); + expect(first.inserted).toBe(3); + expect(await countUnder(tree)).toBe(3); + + // Spot-check one record's shape: type/sha meta + commit-date temporal + + // file list in the content. + const [row] = await sql.unsafe( + `select content, meta from metest_${spaceSlug}.memory + where tree = $1::ltree and content like 'fix: adjust b%'`, + [tree], + ); + expect(row?.meta?.type).toBe("git_commit"); + expect(row?.meta?.sha).toMatch(/^[0-9a-f]{40}$/); + expect(row?.meta?.author_email).toBe("e2e@example.test"); + expect(row?.content).toContain("Files:"); + expect(row?.content).toContain("b.txt (+1 -0)"); + + // 2. Plain re-run: the high-water commit is HEAD → incremental walk of + // an empty range; nothing re-sent, nothing duplicated. + const rerun = await meJson<{ inserted: number; commitsWalked: number }>([ + "import", + "git", + repo, + ]); + expect(rerun.commitsWalked).toBe(0); + expect(rerun.inserted).toBe(0); + expect(await countUnder(tree)).toBe(3); + + // 3. --full re-run: walks everything; deterministic ids make the server + // skip every row (`ON CONFLICT DO NOTHING`). + const full = await meJson<{ + inserted: number; + skipped: number; + commitsWalked: number; + }>(["import", "git", "--full", repo]); + expect(full.commitsWalked).toBe(3); + expect(full.inserted).toBe(0); + expect(full.skipped).toBe(3); + expect(await countUnder(tree)).toBe(3); + + // 4. New work: one regular commit + one body-less merge. The next plain + // run walks only the new range, imports the commit, and drops the + // boilerplate merge. + await git(repo, ["checkout", "-q", "-b", "feat"], undefined); + await commitFile("d.txt", "feat: add d", "2026-05-04T10:00:00Z"); + await git(repo, ["checkout", "-q", "main"]); + await git( + repo, + ["merge", "-q", "--no-ff", "feat", "-m", "Merge branch 'feat'"], + "2026-05-05T10:00:00Z", + ); + const incr = await meJson<{ + inserted: number; + commitsWalked: number; + skippedMerges: number; + range?: string; + }>(["import", "git", repo]); + expect(incr.range).toMatch(/^[0-9a-f]{40}\.\.HEAD$/); + expect(incr.commitsWalked).toBe(2); + expect(incr.inserted).toBe(1); + expect(incr.skippedMerges).toBe(1); + expect(await countUnder(tree)).toBe(4); + + await rm(root, { recursive: true, force: true }); + }); + + test("8e. `me claude init` runs the git step; --skip-git-import suppresses it", async () => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-gitinit-")); + const name = `gitinit${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + await writeFile(join(repo, "x.txt"), "x\n"); + await git(repo, ["add", "."], "2026-05-01T10:00:00Z"); + await git( + repo, + ["commit", "-q", "-m", "feat: initial"], + "2026-05-01T10:00:00Z", + ); + + // --skip-git-import: no commit memories. + const skipped = await me( + ["claude", "init", "--skip-git-import"], + undefined, + repo, + ); + expect(skipped.code, skipped.stderr).toBe(0); + expect(await countUnder(tree)).toBe(0); + + // Plain init (non-interactive baseline) imports the repo's history and + // the CLAUDE.md pointer names the git_history node. + const init = await me(["claude", "init"], undefined, repo); + expect(init.code, init.stderr).toBe(0); + expect(await countUnder(tree)).toBe(1); + const claudeMd = await readFile(join(repo, "CLAUDE.md"), "utf8"); + expect(claudeMd).toContain(`${tree}\``); + + await rm(root, { recursive: true, force: true }); + }); + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { // A minimal Claude Code session transcript on disk. The importer scans // //*.jsonl; the hook reads the file directly. @@ -645,6 +815,32 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( await rm(root, { recursive: true, force: true }); }); + test("9b. `me import` group: no bare default, memories ≡ memory import", async () => { + // Bare `me import` is a group, not the old file-import alias: it prints + // the subcommand list and exits non-zero. + const bare = await me(["import"]); + expect(bare.code).not.toBe(0); + expect(bare.stdout + bare.stderr).toContain("memories"); + + // Old muscle memory `me import ` no longer parses. + const fileArg = await me(["import", "nosuch.md"]); + expect(fileArg.code).not.toBe(0); + expect(fileArg.stderr).toContain("unknown command"); + + // The file importer lives at `me import memories`, with + // `me memory import` as its alias — both write the same records. + const record = (i: number) => + JSON.stringify({ + content: `import group probe ${i}`, + tree: "share.importgroup", + }); + const viaGroup = await meStdin(["import", "memories", "-"], record(1)); + expect(viaGroup.code, viaGroup.stderr).toBe(0); + const viaAlias = await meStdin(["memory", "import", "-"], record(2)); + expect(viaAlias.code, viaAlias.stderr).toBe(0); + expect(await countUnder("share.importgroup")).toBe(2); + }); + test("10. failure modes: bad space and missing auth exit non-zero", async () => { const badSpace = await me(["search", "--fulltext", "fox"], { ME_SPACE: "doesnotexist1", diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index ee941e5..28ca77d 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -3,7 +3,7 @@ * * Capture itself is the import path: the hook reads the session transcript and * runs it through `importTranscriptFile` (packages/cli/importers), so live - * captures and `me import` produce identical memories (tree, ids, `source_*` + * captures and `me import claude` produce identical memories (tree, ids, `source_*` * metadata). This module only resolves the runtime config (bearer + space + * tree root + content mode) and types the slice of the hook event payload we * read. The orchestration lives in `commands/claude.ts` (`me claude hook`). @@ -15,7 +15,7 @@ import { export const DEFAULT_SERVER = "https://api.memory.build"; -/** Per-project sessions leaf, shared with `me import`. */ +/** Per-project sessions leaf, shared with `me import claude`. */ export const SESSIONS_NODE = DEFAULT_SESSIONS_NODE_NAME; /** diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 1e21916..15918c2 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -41,6 +41,7 @@ import { import { createMemoryClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { claudeImporter } from "../importers/claude.ts"; +import { GIT_HISTORY_NODE_NAME } from "../importers/git.ts"; import { DEFAULT_SESSIONS_NODE_NAME, DEFAULT_TREE_ROOT, @@ -52,7 +53,8 @@ import { runAgentMcpInstall, } from "../mcp/agent-install.ts"; import { getOutputFormat } from "../output.ts"; -import { buildAgentImportSubcommand, runAgentImport } from "./import.ts"; +import { createClaudeImportCommand, runAgentImport } from "./import.ts"; +import { runGitImport } from "./import-git.ts"; /** GitHub source for `claude plugin marketplace add`. */ const PLUGIN_MARKETPLACE_SOURCE = "timescale/memory-engine"; @@ -354,7 +356,7 @@ async function runClaudePluginInstall( * Reads the event JSON from stdin for the `transcript_path`, resolves config * from the CLAUDE_PLUGIN_OPTION_* env vars (falling back to the `me login` * session when no api_key is configured), and runs the transcript through - * `importTranscriptFile` — the same parse + write as `me import`, incremental so + * `importTranscriptFile` — the same parse + write as `me import claude`, incremental so * each call only writes messages new since the last. * * Best-effort: logs failures to stderr but always exits 0 so that a hook @@ -409,7 +411,7 @@ function createClaudeHookCommand(): Command { process.exit(0); } - // Import the transcript (incremental; same path as `me import`). + // Import the transcript (incremental; same path as `me import claude`). try { const client = createMemoryClient({ url: config.server, @@ -451,6 +453,7 @@ const DIM_OFF = "\x1b[22m"; */ function buildClaudeMdSection(projectTree: string, space?: string): string { const sessions = `${projectTree}.${DEFAULT_SESSIONS_NODE_NAME}`; + const gitHistory = `${projectTree}.${GIT_HISTORY_NODE_NAME}`; const where = space ? `Memory Engine (space \`${space}\`)` : "Memory Engine"; return [ CLAUDE_MD_START, @@ -462,6 +465,7 @@ function buildClaudeMdSection(projectTree: string, space?: string): string { ` ${projectTree}`, "", `- Captured & imported agent sessions: \`${sessions}\``, + `- Imported git commit history: \`${gitHistory}\``, `- Search them with the \`me_memory_search\` MCP tool (set \`tree\` to`, ` \`${projectTree}\`), or from a shell: \`me search "" --tree ${projectTree}\`.`, "", @@ -559,6 +563,14 @@ const INIT_STEPS: InitStep[] = [ label: "Import existing Claude Code sessions", run: ({ globalOpts }) => runAgentImport(claudeImporter, {}, globalOpts), }, + { + id: "git-import", + optionKey: "skipGitImport", + skipFlag: "--skip-git-import", + skipDescription: "do not import the repo's git commit history", + label: "Import git commit history", + run: ({ globalOpts }) => runGitImport({ skipIfNotRepo: true }, globalOpts), + }, { id: "claude-md", optionKey: "skipClaudeMd", @@ -633,12 +645,6 @@ export function createClaudeCommand(): Command { claude.addCommand(createClaudeInstallCommand()); claude.addCommand(createClaudeInitCommand()); claude.addCommand(createClaudeHookCommand()); - claude.addCommand( - buildAgentImportSubcommand( - "import Claude Code sessions from ~/.claude/projects", - claudeImporter, - true, - ), - ); + claude.addCommand(createClaudeImportCommand()); return claude; } diff --git a/packages/cli/commands/codex.ts b/packages/cli/commands/codex.ts index ba80ea5..c3b3329 100644 --- a/packages/cli/commands/codex.ts +++ b/packages/cli/commands/codex.ts @@ -4,12 +4,11 @@ * - me codex install: register me as an MCP server with Codex CLI */ import { Command } from "commander"; -import { codexImporter } from "../importers/codex.ts"; import { type AgentInstallOptions, runAgentMcpInstall, } from "../mcp/agent-install.ts"; -import { buildAgentImportSubcommand } from "./import.ts"; +import { createCodexImportCommand } from "./import.ts"; function createCodexInstallCommand(): Command { return new Command("install") @@ -36,11 +35,6 @@ function createCodexInstallCommand(): Command { export function createCodexCommand(): Command { const codex = new Command("codex").description("Codex CLI integration"); codex.addCommand(createCodexInstallCommand()); - codex.addCommand( - buildAgentImportSubcommand( - "import Codex sessions from ~/.codex/sessions and archived_sessions", - codexImporter, - ), - ); + codex.addCommand(createCodexImportCommand()); return codex; } diff --git a/packages/cli/commands/import-git.test.ts b/packages/cli/commands/import-git.test.ts new file mode 100644 index 0000000..894eb57 --- /dev/null +++ b/packages/cli/commands/import-git.test.ts @@ -0,0 +1,62 @@ +/** + * Tests for `me import git` option assembly. + */ +import { describe, expect, test } from "bun:test"; +import { buildGitImportOptions } from "./import-git.ts"; + +describe("buildGitImportOptions", () => { + test("applies defaults", () => { + const opts = buildGitImportOptions({}); + expect(opts).toEqual({ + repo: undefined, + branch: undefined, + since: undefined, + until: undefined, + maxCount: undefined, + full: false, + merges: true, + fileList: true, + treeRoot: "share.projects", + dryRun: false, + verbose: false, + skipIfNotRepo: false, + }); + }); + + test("maps flags through", () => { + const opts = buildGitImportOptions( + { + branch: "main", + since: "2 weeks ago", + until: "2026-01-01", + maxCount: 100, + full: true, + merges: false, + fileList: false, + treeRoot: "~/work", + dryRun: true, + verbose: true, + skipIfNotRepo: true, + }, + "/some/repo", + ); + expect(opts.repo).toBe("/some/repo"); + expect(opts.branch).toBe("main"); + expect(opts.since).toBe("2 weeks ago"); + expect(opts.until).toBe("2026-01-01"); + expect(opts.maxCount).toBe(100); + expect(opts.full).toBe(true); + expect(opts.merges).toBe(false); + expect(opts.fileList).toBe(false); + expect(opts.treeRoot).toBe("~/work"); + expect(opts.dryRun).toBe(true); + expect(opts.verbose).toBe(true); + expect(opts.skipIfNotRepo).toBe(true); + }); + + test("rejects an invalid --tree-root", () => { + expect(() => buildGitImportOptions({ treeRoot: "bad path!" })).toThrow( + /Invalid --tree-root/, + ); + }); +}); diff --git a/packages/cli/commands/import-git.ts b/packages/cli/commands/import-git.ts new file mode 100644 index 0000000..b5495d8 --- /dev/null +++ b/packages/cli/commands/import-git.ts @@ -0,0 +1,356 @@ +/** + * `me import git` — import a repo's commit history as memories. + * + * One memory per commit (message + capped changed-file list) under + * `..git_history`, with the commit date as the + * memory's temporal and a deterministic id keyed by `(tree, sha)` — so + * re-runs are idempotent (existing commits become server-side skips). + * + * Re-runs are also incremental: the newest already-imported commit is looked + * up server-side (one search) and, when it is an ancestor of the target rev, + * only `..` is walked. Any doubt (force-push, other branch, + * explicit bounds) falls back to the full walk, which deterministic ids make + * safe. `--full` forces the full walk. + */ +import { resolve } from "node:path"; +import * as clack from "@clack/prompts"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; +import { Command, InvalidArgumentError } from "commander"; +import { batchCreateChunked } from "../chunk.ts"; +import type { MemoryClient } from "../client.ts"; +import { resolveCredentials } from "../credentials.ts"; +import { + buildCommitMemory, + GIT_HISTORY_NODE_NAME, + isAncestor, + mergeSkipReason, + walkGitLog, +} from "../importers/git.ts"; +import { + createProgressReporter, + DEFAULT_TREE_ROOT, + dedupByMemoryId, +} from "../importers/index.ts"; +import { SlugRegistry } from "../importers/slug.ts"; +import { getOutputFormat, output } from "../output.ts"; +import { + buildMemoryClient, + handleError, + requireMemoryAuth, + requireSpace, +} from "../util.ts"; +import { VALID_TREE_ROOT_RE } from "./import.ts"; +import { computeSkippedIds } from "./memory-import.ts"; + +/** Parsed options for one git import run. */ +export interface GitImportOptions { + /** Repo path (any directory inside the repo). Default: cwd. */ + repo?: string; + /** Rev to walk (branch, tag, sha). Default: HEAD. */ + branch?: string; + /** `git log --since` bound (git accepts ISO or approxidate). */ + since?: string; + /** `git log --until` bound. */ + until?: string; + /** Cap on walked commits. */ + maxCount?: number; + /** Force the full walk (skip the incremental high-water lookup). */ + full?: boolean; + /** False (via --no-merges) drops all merge commits. */ + merges?: boolean; + /** False (via --no-file-list) omits the changed-file list from content. */ + fileList?: boolean; + /** Tree root under which `.git_history` is placed. */ + treeRoot?: string; + /** Report without writing. */ + dryRun?: boolean; + /** Per-commit progress output. */ + verbose?: boolean; + /** + * Soft-skip (info, success) when the target isn't a git repo — used by + * `me claude init`, which runs in arbitrary directories. + */ + skipIfNotRepo?: boolean; +} + +/** Validate raw Commander opts into a typed option set. */ +export function buildGitImportOptions( + opts: Record, + repoArg?: string, +): GitImportOptions { + const treeRoot = + typeof opts.treeRoot === "string" ? opts.treeRoot : DEFAULT_TREE_ROOT; + if (!VALID_TREE_ROOT_RE.test(treeRoot)) { + throw new Error( + `Invalid --tree-root: '${treeRoot}'. Use ltree labels ([A-Za-z0-9_-]) separated by '.' or '/', with an optional leading '~' for your home.`, + ); + } + return { + repo: repoArg, + branch: typeof opts.branch === "string" ? opts.branch : undefined, + since: typeof opts.since === "string" ? opts.since : undefined, + until: typeof opts.until === "string" ? opts.until : undefined, + maxCount: typeof opts.maxCount === "number" ? opts.maxCount : undefined, + full: opts.full === true, + merges: opts.merges !== false, + fileList: opts.fileList !== false, + treeRoot, + dryRun: opts.dryRun === true, + verbose: opts.verbose === true, + skipIfNotRepo: opts.skipIfNotRepo === true, + }; +} + +/** Structured result of one run (also the --json/--yaml output shape). */ +interface GitImportResult { + repo: string; + remote?: string; + tree: string; + rev: string; + /** The incremental range actually walked, when one was used. */ + range?: string; + dryRun: boolean; + commitsWalked: number; + inserted: number; + /** Already present server-side (idempotent re-import). */ + skipped: number; + /** Merge commits dropped by the boilerplate rule. */ + skippedMerges: number; + failed: number; + errors: Array<{ sha: string; error: string }>; +} + +/** + * Newest already-imported commit sha under `tree`, or null. Unranked search + * returns newest-first by id, and git ids encode the commit date — so one + * `limit: 1` search yields the high-water commit. + */ +async function searchHighWaterSha( + engine: MemoryClient, + tree: string, +): Promise { + const res = await engine.memory.search({ + tree, + meta: { type: "git_commit" }, + limit: 1, + }); + const sha = res.results[0]?.meta.sha; + return typeof sha === "string" && /^[0-9a-f]{40}$/.test(sha) ? sha : null; +} + +/** + * Run one git history import end-to-end and render the outcome. Exported so + * `me claude init` can run it as a setup step, reusing the same + * auth/option/render path as the standalone `me import git`. + */ +export async function runGitImport( + rawOpts: Record, + globalOpts: Record, + repoArg?: string, +): Promise { + const creds = resolveCredentials( + typeof globalOpts.server === "string" ? globalOpts.server : undefined, + ); + const fmt = getOutputFormat(globalOpts); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); + + let opts: GitImportOptions; + try { + opts = buildGitImportOptions(rawOpts, repoArg); + } catch (error) { + handleError(error, fmt); + } + + const repoPath = resolve(opts.repo ?? process.cwd()); + const { slug, gitRoot, gitRemote } = await new SlugRegistry().resolve( + repoPath, + ); + if (!gitRoot) { + if (opts.skipIfNotRepo) { + if (fmt === "text") { + clack.log.info( + `${repoPath} is not a git repository — skipping git history import`, + ); + } + return; + } + handleError(new Error(`${repoPath} is not a git repository`), fmt); + } + + const tree = `${opts.treeRoot}.${slug}.${GIT_HISTORY_NODE_NAME}`; + const rev = opts.branch ?? "HEAD"; + const engine = buildMemoryClient(creds); + + // Incremental fast path: only when nothing narrows the walk explicitly. + const explicitBounds = + opts.full || + opts.since !== undefined || + opts.until !== undefined || + opts.maxCount !== undefined; + let range: string | undefined; + if (!explicitBounds) { + try { + const highWater = await searchHighWaterSha(engine, tree); + if (highWater && (await isAncestor(gitRoot, highWater, rev))) { + range = `${highWater}..${rev}`; + } + } catch (error) { + handleError(error, fmt); + } + } + + const progress = + fmt === "text" ? createProgressReporter(process.stderr) : undefined; + progress?.start(); + + const importedAt = new Date().toISOString(); + const planned: Array<{ memoryId: string; payload: MemoryCreateParams }> = []; + let commitsWalked = 0; + let skippedMerges = 0; + let failed = 0; + const errors: Array<{ sha: string; error: string }> = []; + + try { + for await (const commit of walkGitLog(gitRoot, { + rev, + range, + since: opts.since, + until: opts.until, + maxCount: opts.maxCount, + noMerges: opts.merges === false, + })) { + commitsWalked++; + progress?.process(`${commit.sha.slice(0, 8)} ${commit.subject}`); + if (mergeSkipReason(commit) !== null) { + skippedMerges++; + continue; + } + const built = buildCommitMemory(commit, { + tree, + projectSlug: slug, + gitRemote, + fileList: opts.fileList !== false, + importedAt, + }); + if ("error" in built) { + failed++; + errors.push({ sha: commit.sha, error: built.error }); + continue; + } + planned.push({ memoryId: built.id as string, payload: built }); + if (opts.verbose && fmt === "text") { + const line = ` ${commit.sha.slice(0, 8)} ${commit.subject}`; + if (progress) progress.log(line); + else console.log(line); + } + } + } catch (error) { + progress?.stop(); + handleError(error, fmt); + } + + const { unique } = dedupByMemoryId(planned); + + let inserted = 0; + let skipped = 0; + if (opts.dryRun) { + inserted = unique.length; + } else if (unique.length > 0) { + const submitted = unique.map((p) => p.memoryId); + const result = await batchCreateChunked( + engine, + unique.map((p) => p.payload), + ); + inserted = result.insertedIds.length; + const failedSet = new Set(result.failedIds); + skipped = computeSkippedIds(submitted, result.insertedIds).filter( + (id) => !failedSet.has(id), + ).length; + for (const e of result.errors) { + failed += e.itemCount; + errors.push({ sha: `chunk ${e.chunkIndex}`, error: e.error }); + } + } + progress?.stop(); + + const structured: GitImportResult = { + repo: gitRoot, + remote: gitRemote, + tree, + rev, + range, + dryRun: opts.dryRun === true, + commitsWalked, + inserted, + skipped, + skippedMerges, + failed, + errors, + }; + + output(structured, fmt, () => { + const verb = opts.dryRun ? "Would import" : "Imported"; + clack.log.success( + `${verb} ${inserted} new, ${skipped} already present, ${failed} failed ` + + `commits into ${tree}`, + ); + if (range) console.log(` Incremental walk: ${range}`); + console.log(` Walked ${commitsWalked} commits from ${gitRoot} (${rev})`); + if (skippedMerges > 0) { + console.log(` Skipped ${skippedMerges} boilerplate merge commit(s)`); + } + for (const e of errors) { + console.log(` ✗ ${e.sha}: ${e.error}`); + } + }); + + if (failed > 0 && inserted === 0) process.exit(2); + if (failed > 0) process.exit(1); +} + +/** Parse `--max-count` into a positive integer. */ +function parseMaxCount(value: string): number { + const n = Number.parseInt(value, 10); + if (!Number.isInteger(n) || n <= 0) { + throw new InvalidArgumentError("must be a positive integer"); + } + return n; +} + +/** `me import git` subcommand factory. */ +export function createGitImportCommand(): Command { + return new Command("git") + .description("import a repo's git commit history as memories") + .argument("[repo]", "path inside the repo to import (default: cwd)") + .option("--branch ", "branch/tag/rev to walk (default: HEAD)") + .option( + "--since ", + "only commits at/after this date (any format git accepts)", + ) + .option("--until ", "only commits at/before this date") + .option( + "--max-count ", + "import at most this many recent commits", + parseMaxCount, + ) + .option( + "--full", + "walk the full history (skip the incremental high-water lookup)", + ) + .option("--no-merges", "drop all merge commits") + .option("--no-file-list", "omit the changed-file list from commit memories") + .option( + "--tree-root ", + `tree root under which '.${GIT_HISTORY_NODE_NAME}' is placed (default: ${DEFAULT_TREE_ROOT})`, + ) + .option( + "--dry-run", + "parse and report what would be imported without writing", + ) + .option("-v, --verbose", "per-commit progress output") + .action(async (repoArg: string | undefined, opts, cmdRef) => { + const globalOpts = cmdRef.optsWithGlobals(); + await runGitImport(opts, globalOpts, repoArg); + }); +} diff --git a/packages/cli/commands/import-group.ts b/packages/cli/commands/import-group.ts new file mode 100644 index 0000000..5842159 --- /dev/null +++ b/packages/cli/commands/import-group.ts @@ -0,0 +1,41 @@ +/** + * `me import` — the umbrella group for getting data into Memory Engine. + * + * One subcommand per source: + * + * me import memories file records (md/yaml/json/ndjson) + * me import claude Claude Code sessions + * me import codex Codex sessions + * me import opencode OpenCode sessions + * me import git [repo] git commit history + * + * There is deliberately no bare default: `me import ` does not parse. + * The pre-group spellings stay registered as aliases built from the same + * factories — `me memory import` (⇒ memories) and `me import` + * (⇒ claude/codex/opencode) — so adding a source here is one subcommand, + * never a new top-level command group. + */ +import { Command } from "commander"; +import { + createClaudeImportCommand, + createCodexImportCommand, + createOpenCodeImportCommand, +} from "./import.ts"; +import { createGitImportCommand } from "./import-git.ts"; +import { createMemoryImportCommand } from "./memory-import.ts"; + +export function createImportCommand(): Command { + const imp = new Command("import").description( + "import memories, agent sessions, and git history", + ); + imp.addCommand(createMemoryImportCommand("memories")); + imp.addCommand(createClaudeImportCommand("claude")); + imp.addCommand(createCodexImportCommand("codex")); + imp.addCommand(createOpenCodeImportCommand("opencode")); + imp.addCommand(createGitImportCommand()); + imp.addHelpText( + "after", + "\nTo import memory files (the old `me import `), use: me import memories ", + ); + return imp; +} diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index 8b84e47..e32ced8 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -1,12 +1,13 @@ /** - * Shared helpers for the per-agent `import` subcommands. + * Shared helpers for the agent-session import subcommands. * - * Each agent command group (`me claude`, `me codex`, `me opencode`) adds its - * own `import` subcommand via `buildAgentImportSubcommand`. Each source-native - * message becomes one memory, stored under - * `..`. + * Each agent importer is exposed twice: canonically under the import group + * (`me import claude|codex|opencode`) and as an alias under its agent command + * group (`me claude import`, …) — both built from the same per-tool factory + * (`createClaudeImportCommand`, …). Each source-native message becomes one + * memory, stored under `..`. * - * Shared flags across every `import` subcommand: + * Shared flags across every agent import subcommand: * --source override default source directory * --project only import sessions with this cwd (or a child) * --since only sessions started at/after this timestamp @@ -25,6 +26,8 @@ import * as clack from "@clack/prompts"; import { Command } from "commander"; import { resolveCredentials } from "../credentials.ts"; +import { claudeImporter } from "../importers/claude.ts"; +import { codexImporter } from "../importers/codex.ts"; import { createProgressReporter, DEFAULT_SESSIONS_NODE_NAME, @@ -34,6 +37,7 @@ import { runImport, type WriteOptions, } from "../importers/index.ts"; +import { opencodeImporter } from "../importers/opencode.ts"; import type { ImporterOptions } from "../importers/types.ts"; import { getOutputFormat, output } from "../output.ts"; import { @@ -44,12 +48,12 @@ import { } from "../util.ts"; // Default capture layout (share.projects..agent_sessions) lives in the -// importers module so `me import` and the Claude Code hook share one source. +// importers module so `me import ` and the Claude Code hook share one source. // Lenient user-facing tree-path input (matches the protocol's treePathSchema): // labels [A-Za-z0-9_-], `.` or `/` separators, optional leading `~` (home). The // server normalizes + authoritatively validates; this is a fast pre-check so a // `--tree-root ~/work` or `~` lands in the caller's home instead of being rejected. -const VALID_TREE_ROOT_RE = /^[A-Za-z0-9_~./-]+$/; +export const VALID_TREE_ROOT_RE = /^[A-Za-z0-9_~./-]+$/; const VALID_TREE_LABEL_RE = /^[a-z0-9_]+$/; /** Build a Commander option set shared by every subcommand. */ @@ -312,16 +316,18 @@ function renderResult( } /** - * Build an `import` subcommand bound to a specific importer. Each agent - * command group (`me claude`, `me codex`, `me opencode`) calls this to add - * its own `import` subcommand. + * Build a subcommand bound to a specific importer. Each importer is + * registered twice: under the `me import` group as `me import ` (its + * canonical spelling) and under the agent's command group as the + * `me import` alias — hence the `name` parameter. */ -export function buildAgentImportSubcommand( +function buildAgentImportSubcommand( description: string, importer: Importer, includeSidechainsFlag = false, + name = "import", ): Command { - const cmd = new Command("import").description(description); + const cmd = new Command(name).description(description); addCommonOptions(cmd, includeSidechainsFlag); cmd.action(async (opts, cmdRef) => { const globalOpts = cmdRef.optsWithGlobals(); @@ -329,3 +335,35 @@ export function buildAgentImportSubcommand( }); return cmd; } + +/** + * Per-tool import subcommand factories. Each owns its importer wiring + + * description in one place so both registrations (`me import ` and the + * `me import` alias) stay identical. + */ +export function createClaudeImportCommand(name = "import"): Command { + return buildAgentImportSubcommand( + "import Claude Code sessions from ~/.claude/projects", + claudeImporter, + true, + name, + ); +} + +export function createCodexImportCommand(name = "import"): Command { + return buildAgentImportSubcommand( + "import Codex sessions from ~/.codex/sessions and archived_sessions", + codexImporter, + false, + name, + ); +} + +export function createOpenCodeImportCommand(name = "import"): Command { + return buildAgentImportSubcommand( + "import OpenCode sessions from ~/.local/share/opencode/storage", + opencodeImporter, + false, + name, + ); +} diff --git a/packages/cli/commands/memory-import.test.ts b/packages/cli/commands/memory-import.test.ts index 74e78ba..b317ab1 100644 --- a/packages/cli/commands/memory-import.test.ts +++ b/packages/cli/commands/memory-import.test.ts @@ -1,5 +1,5 @@ /** - * Tests for `me memory import` helpers. + * Tests for `me import memories` (alias `me memory import`) helpers. * * The skip-detection helper exists because `engine.memory.batchCreate` * silently drops conflicting ids (post-#64). Memory import — unlike pack diff --git a/packages/cli/commands/memory-import.ts b/packages/cli/commands/memory-import.ts index 106cdc1..8894ebd 100644 --- a/packages/cli/commands/memory-import.ts +++ b/packages/cli/commands/memory-import.ts @@ -89,8 +89,8 @@ export function computeSkippedIds( return explicitIds.filter((id) => !inserted.has(id)); } -export function createMemoryImportCommand(): Command { - return new Command("import") +export function createMemoryImportCommand(name = "import"): Command { + return new Command(name) .description("import memories from files or stdin") .argument("[files...]", "files to import (use - for stdin)") .option("--format ", "override format detection (md|yaml|json)") diff --git a/packages/cli/commands/memory.ts b/packages/cli/commands/memory.ts index 2c77c54..d5a59e9 100644 --- a/packages/cli/commands/memory.ts +++ b/packages/cli/commands/memory.ts @@ -921,7 +921,12 @@ export function createMemoryCommand(): Command { /** * The memory subcommands as top-level aliases (`me search`, `me create`, …) so * the `memory` word is optional for the common data-plane operations. + * + * `import` is excluded: the top-level `import` name belongs to the + * `me import` source group (see commands/import-group.ts), where the file + * importer lives as `me import memories`. `me memory import` remains its + * alias. */ export function createMemoryAliasCommands(): Command[] { - return memorySubcommands(); + return memorySubcommands().filter((c) => c.name() !== "import"); } diff --git a/packages/cli/commands/opencode.ts b/packages/cli/commands/opencode.ts index 9989576..b31a64a 100644 --- a/packages/cli/commands/opencode.ts +++ b/packages/cli/commands/opencode.ts @@ -4,12 +4,11 @@ * - me opencode install: register me as an MCP server with OpenCode */ import { Command } from "commander"; -import { opencodeImporter } from "../importers/opencode.ts"; import { type AgentInstallOptions, runAgentMcpInstall, } from "../mcp/agent-install.ts"; -import { buildAgentImportSubcommand } from "./import.ts"; +import { createOpenCodeImportCommand } from "./import.ts"; function createOpenCodeInstallCommand(): Command { return new Command("install") @@ -36,11 +35,6 @@ function createOpenCodeInstallCommand(): Command { export function createOpenCodeCommand(): Command { const opencode = new Command("opencode").description("OpenCode integration"); opencode.addCommand(createOpenCodeInstallCommand()); - opencode.addCommand( - buildAgentImportSubcommand( - "import OpenCode sessions from ~/.local/share/opencode/storage", - opencodeImporter, - ), - ); + opencode.addCommand(createOpenCodeImportCommand()); return opencode; } diff --git a/packages/cli/importers/git.test.ts b/packages/cli/importers/git.test.ts new file mode 100644 index 0000000..e2cea99 --- /dev/null +++ b/packages/cli/importers/git.test.ts @@ -0,0 +1,287 @@ +/** + * Tests for the git history importer: the streaming `git log` parser and + * the per-commit memory builder. Fixture strings mirror the byte layout + * git actually emits for + * `--numstat --pretty=format:%x01%H%x00…%B%x00` (verified empirically): + * + * \x01\0\0\0\0\0\0\0\n\n\n + * + * Merge commits emit no numstat lines; a root commit has an empty parents + * field; the final record ends at EOF without trailing separators. + */ +import { describe, expect, test } from "bun:test"; +import { + BODY_BYTES_CAP, + buildCommitMemory, + type CommitMemoryContext, + FILE_LIST_CAP, + type GitCommit, + GitLogParser, + mergeSkipReason, +} from "./git.ts"; + +const SHA_A = "a".repeat(40); +const SHA_B = "b".repeat(40); +const SHA_C = "c".repeat(40); + +/** Build one raw log record in the observed wire layout. */ +function rec(opts: { + sha?: string; + authorName?: string; + authorEmail?: string; + authorDate?: string; + commitDate?: string; + parents?: string; + body?: string; + numstat?: string[]; +}): string { + const fields = [ + opts.sha ?? SHA_A, + opts.authorName ?? "Ada", + opts.authorEmail ?? "ada@example.com", + opts.authorDate ?? "2026-01-02T03:04:05+02:00", + opts.commitDate ?? "2026-01-02T03:04:06+02:00", + opts.parents ?? SHA_B, + opts.body ?? "subject line\n", + ].join("\x00"); + const tail = + opts.numstat && opts.numstat.length > 0 + ? `\n${opts.numstat.join("\n")}\n\n` + : "\n"; + return `\x01${fields}\x00${tail}`; +} + +/** Parse a full log text in one push + end. */ +function parseAll(text: string): GitCommit[] { + const parser = new GitLogParser(); + return [...parser.push(text), ...parser.end()]; +} + +describe("GitLogParser", () => { + test("parses a single commit with files", () => { + const commits = parseAll( + rec({ + body: "fix: a thing\n\nlonger explanation\nover two lines\n", + numstat: ["10\t2\tsrc/a.ts", "0\t5\tsrc/b.ts"], + }), + ); + expect(commits).toHaveLength(1); + const c = commits[0]; + expect(c?.sha).toBe(SHA_A); + expect(c?.authorName).toBe("Ada"); + expect(c?.authorEmail).toBe("ada@example.com"); + expect(c?.authorDate).toBe("2026-01-02T03:04:05+02:00"); + expect(c?.commitDate).toBe("2026-01-02T03:04:06+02:00"); + expect(c?.parents).toEqual([SHA_B]); + expect(c?.subject).toBe("fix: a thing"); + expect(c?.body).toBe("longer explanation\nover two lines"); + expect(c?.files).toEqual([ + { path: "src/a.ts", insertions: 10, deletions: 2 }, + { path: "src/b.ts", insertions: 0, deletions: 5 }, + ]); + }); + + test("parses multiple records, including a final record at EOF", () => { + const commits = parseAll( + rec({ sha: SHA_A, numstat: ["1\t0\ta.txt"] }) + + rec({ sha: SHA_B, parents: "", body: "first\n" }), + ); + expect(commits.map((c) => c.sha)).toEqual([SHA_A, SHA_B]); + // Root commit: empty parents field → no parents. + expect(commits[1]?.parents).toEqual([]); + }); + + test("merge commits carry two parents and no files", () => { + const commits = parseAll( + rec({ parents: `${SHA_B} ${SHA_C}`, body: "Merge branch 'x'\n" }), + ); + expect(commits[0]?.parents).toEqual([SHA_B, SHA_C]); + expect(commits[0]?.files).toEqual([]); + }); + + test("handles rename and binary numstat lines", () => { + const commits = parseAll( + rec({ + numstat: ["3\t1\tsrc/{old => new}/mod.ts", "-\t-\tassets/logo.png"], + }), + ); + expect(commits[0]?.files).toEqual([ + { path: "src/{old => new}/mod.ts", insertions: 3, deletions: 1 }, + { path: "assets/logo.png", insertions: null, deletions: null }, + ]); + }); + + test("a \\x01 inside a message body does not start a new record", () => { + const commits = parseAll( + rec({ sha: SHA_A, body: "subject\n\nweird \x01 control char\n" }) + + rec({ sha: SHA_B }), + ); + expect(commits).toHaveLength(2); + expect(commits[0]?.body).toBe("weird \x01 control char"); + }); + + test("reassembles records split across arbitrary chunk boundaries", () => { + const full = + rec({ sha: SHA_A, numstat: ["1\t0\ta.txt"] }) + + rec({ sha: SHA_B, body: "two\n" }) + + rec({ sha: SHA_C, parents: "", body: "three\n" }); + // Split mid-header, mid-field, and mid-numstat to stress the buffer. + for (const chunkSize of [1, 7, 41, 64]) { + const parser = new GitLogParser(); + const commits: GitCommit[] = []; + for (let i = 0; i < full.length; i += chunkSize) { + commits.push(...parser.push(full.slice(i, i + chunkSize))); + } + commits.push(...parser.end()); + expect(commits.map((c) => c.sha)).toEqual([SHA_A, SHA_B, SHA_C]); + } + }); + + test("empty input yields nothing", () => { + const parser = new GitLogParser(); + expect(parser.push("")).toEqual([]); + expect(parser.end()).toEqual([]); + }); + + test("throws on a malformed record", () => { + expect(() => parseAll("\x01not-a-sha\x00rest")).toThrow(/malformed/); + }); +}); + +/** A parsed commit for builder tests. */ +function commit(overrides: Partial = {}): GitCommit { + return { + sha: SHA_A, + authorName: "Ada", + authorEmail: "ada@example.com", + authorDate: "2026-01-02T03:04:05+02:00", + commitDate: "2026-01-02T03:04:06+02:00", + parents: [SHA_B], + subject: "fix: a thing", + body: "details here", + files: [{ path: "src/a.ts", insertions: 10, deletions: 2 }], + ...overrides, + }; +} + +function ctx( + overrides: Partial = {}, +): CommitMemoryContext { + return { + tree: "share.projects.demo.git_history", + projectSlug: "demo", + gitRemote: "git@github.com:org/demo.git", + fileList: true, + importedAt: "2026-06-10T00:00:00.000Z", + ...overrides, + }; +} + +describe("buildCommitMemory", () => { + test("builds content, meta, temporal, and a deterministic id", () => { + const built = buildCommitMemory(commit(), ctx()); + if ("error" in built) throw new Error(built.error); + expect(built.content).toBe( + "fix: a thing\n\ndetails here\n\nFiles:\n src/a.ts (+10 -2)", + ); + expect(built.tree).toBe("share.projects.demo.git_history"); + // Commit date, normalized to UTC. + expect(built.temporal).toEqual({ start: "2026-01-02T01:04:06.000Z" }); + expect(built.meta).toEqual({ + type: "git_commit", + sha: SHA_A, + source_project_slug: "demo", + source_git_repo: "git@github.com:org/demo.git", + author_name: "Ada", + author_email: "ada@example.com", + author_date: "2026-01-02T03:04:05+02:00", + commit_date: "2026-01-02T03:04:06+02:00", + files_changed: 1, + insertions: 10, + deletions: 2, + imported_at: "2026-06-10T00:00:00.000Z", + importer_version: "1", + }); + + // Deterministic: same inputs → same id; different tree → different id. + const again = buildCommitMemory(commit(), ctx()); + if ("error" in again) throw new Error(again.error); + expect(again.id).toBe(built.id); + const moved = buildCommitMemory(commit(), ctx({ tree: "share.other" })); + if ("error" in moved) throw new Error(moved.error); + expect(moved.id).not.toBe(built.id); + }); + + test("omits remote and merge marker when absent, sets them when present", () => { + const plain = buildCommitMemory(commit(), ctx({ gitRemote: undefined })); + if ("error" in plain) throw new Error(plain.error); + expect(plain.meta).not.toContainKey("source_git_repo"); + expect(plain.meta).not.toContainKey("is_merge"); + + const merge = buildCommitMemory(commit({ parents: [SHA_B, SHA_C] }), ctx()); + if ("error" in merge) throw new Error(merge.error); + expect(merge.meta?.is_merge).toBe(true); + }); + + test("renders binary files and caps the file list", () => { + const files = Array.from({ length: FILE_LIST_CAP + 3 }, (_, i) => ({ + path: `f${i}.ts`, + insertions: 1, + deletions: 0, + })); + files[0] = { path: "img.png", insertions: null, deletions: null } as never; + const built = buildCommitMemory(commit({ files }), ctx()); + if ("error" in built) throw new Error(built.error); + expect(built.content).toContain(" img.png (binary)"); + expect(built.content).toContain(` … and 3 more files`); + // Binary files don't contribute to line counts. + expect(built.meta?.insertions).toBe(FILE_LIST_CAP + 2); + expect(built.meta?.files_changed).toBe(FILE_LIST_CAP + 3); + }); + + test("omits the file list when disabled", () => { + const built = buildCommitMemory(commit(), ctx({ fileList: false })); + if ("error" in built) throw new Error(built.error); + expect(built.content).toBe("fix: a thing\n\ndetails here"); + }); + + test("truncates oversized bodies on a byte budget", () => { + const built = buildCommitMemory( + commit({ body: "x".repeat(BODY_BYTES_CAP + 1000), files: [] }), + ctx(), + ); + if ("error" in built) throw new Error(built.error); + expect(built.content).toContain("…[truncated]"); + expect(Buffer.byteLength(built.content, "utf8")).toBeLessThan( + BODY_BYTES_CAP + 200, + ); + }); + + test("reports an error for an unparsable commit date", () => { + const built = buildCommitMemory( + commit({ commitDate: "not-a-date" }), + ctx(), + ); + expect(built).toEqual({ error: "invalid commit date: not-a-date" }); + }); +}); + +describe("mergeSkipReason", () => { + test("non-merge commits are never skipped", () => { + expect(mergeSkipReason(commit({ body: "" }))).toBeNull(); + }); + + test("body-less merges are boilerplate", () => { + expect(mergeSkipReason(commit({ parents: [SHA_B, SHA_C], body: "" }))).toBe( + "merge_boilerplate", + ); + }); + + test("merges with a body (PR merges) are kept", () => { + expect( + mergeSkipReason( + commit({ parents: [SHA_B, SHA_C], body: "Fix auth refresh (#42)" }), + ), + ).toBeNull(); + }); +}); diff --git a/packages/cli/importers/git.ts b/packages/cli/importers/git.ts new file mode 100644 index 0000000..c17ab85 --- /dev/null +++ b/packages/cli/importers/git.ts @@ -0,0 +1,372 @@ +/** + * Git history importer — walks `git log` and turns each commit into one + * memory under `..git_history`. + * + * Identity: a deterministic UUIDv7 keyed by `git::` with the + * commit date as the timestamp half, so re-imports collide server-side + * (`ON CONFLICT (id) DO NOTHING`) and become no-op skips — no cursor or + * client-side state. Incremental runs (see commands/import-git.ts) only + * narrow the walk; correctness never depends on them. + * + * The walk is one streamed `git log` invocation with NUL-separated fields: + * + * %x01 %H %x00 %an %x00 %ae %x00 %aI %x00 %cI %x00 %P %x00 %B %x00 + * + * followed by `--numstat` lines until the next record. Git forbids NUL in + * commit messages, so the field splits are unambiguous; records are anchored + * by `\x01` + 40-hex sha + NUL so a `\x01` inside a message body can't start + * a record. Output is streamed through an incremental parser, so repos of + * any size walk in constant memory. + */ + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; +import { deterministicUuidV7 } from "./uuid.ts"; + +const execFileAsync = promisify(execFile); + +/** Per-project tree node holding imported commits (next to agent_sessions). */ +export const GIT_HISTORY_NODE_NAME = "git_history"; + +/** + * Version tag stored in `meta.importer_version`. Reserved for a future + * re-render path (cf. IMPORTER_VERSION in importers/index.ts); for now a + * bump only marks newly-written records. + */ +export const GIT_IMPORTER_VERSION = "1"; + +/** Max file lines rendered into a commit memory's `Files:` block. */ +export const FILE_LIST_CAP = 50; + +/** Max body bytes rendered into a commit memory before truncation. */ +export const BODY_BYTES_CAP = 64 * 1024; + +/** One changed file from a `--numstat` line. */ +export interface GitFileChange { + /** Path as git prints it (renames keep the `{old => new}` form). */ + path: string; + /** Added lines, or null for binary files. */ + insertions: number | null; + /** Deleted lines, or null for binary files. */ + deletions: number | null; +} + +/** One parsed commit from the log walk. */ +export interface GitCommit { + sha: string; + authorName: string; + authorEmail: string; + /** ISO 8601 author date (%aI). */ + authorDate: string; + /** ISO 8601 committer date (%cI). */ + commitDate: string; + /** Parent shas; length >= 2 marks a merge commit. */ + parents: string[]; + /** First line of the message. */ + subject: string; + /** Message after the subject (trimmed; may be empty). */ + body: string; + files: GitFileChange[]; +} + +/** Record start marker + the number of NUL-separated fields before the tail. */ +const RECORD_START = "\x01"; +const FIELD_COUNT = 7; +/** `\x01` + 40-hex sha + NUL — what a genuine record header looks like. */ +// biome-ignore lint/suspicious/noControlCharactersInRegex: the log wire format uses \x01/\x00 separators by design +const RECORD_HEADER_RE = /^\x01[0-9a-f]{40}\x00/; +/** A `--numstat` line: added/deleted counts (or `-` for binary) + path. */ +const NUMSTAT_LINE_RE = /^(\d+|-)\t(\d+|-)\t(.+)$/; + +/** + * Incremental parser for the custom `git log` format above. Feed decoded + * stdout text via `push` (returns the records completed by that chunk) and + * call `end` for the final record. Pure — unit-testable without git. + */ +export class GitLogParser { + private buf = ""; + + push(text: string): GitCommit[] { + this.buf += text; + const out: GitCommit[] = []; + // A record is complete once the NEXT record's header is visible. + for (;;) { + const next = findNextHeader(this.buf, 1); + if (next === -1) break; + out.push(parseRecord(this.buf.slice(0, next))); + this.buf = this.buf.slice(next); + } + return out; + } + + end(): GitCommit[] { + const rest = this.buf; + this.buf = ""; + if (rest.trim().length === 0) return []; + return [parseRecord(rest)]; + } +} + +/** Find the next genuine record header at or after `from` (-1 if none). */ +function findNextHeader(buf: string, from: number): number { + let i = from; + for (;;) { + const at = buf.indexOf(RECORD_START, i); + if (at === -1) return -1; + if (RECORD_HEADER_RE.test(buf.slice(at, at + 42))) return at; + i = at + 1; + } +} + +/** Parse one complete record (from its `\x01` up to the next header). */ +function parseRecord(record: string): GitCommit { + if (!RECORD_HEADER_RE.test(record.slice(0, 42))) { + throw new Error( + `malformed git log record: ${JSON.stringify(record.slice(0, 60))}`, + ); + } + // Split off the 7 NUL-separated fields; the remainder is the numstat tail. + const parts = record.slice(1).split("\x00"); + if (parts.length < FIELD_COUNT + 1) { + throw new Error( + `malformed git log record for ${parts[0]}: expected ${FIELD_COUNT} fields`, + ); + } + const [sha, authorName, authorEmail, authorDate, commitDate, parentsRaw] = + parts; + // The body is everything between the 6th NUL and the 7th — but a body + // cannot contain NUL, so it is exactly parts[6]. + const bodyRaw = parts[6] ?? ""; + const tail = parts.slice(FIELD_COUNT).join("\x00"); + + const files: GitFileChange[] = []; + for (const line of tail.split("\n")) { + const m = NUMSTAT_LINE_RE.exec(line); + if (!m) continue; + const [, ins, del, path] = m; + files.push({ + path: path ?? "", + insertions: ins === "-" ? null : Number(ins), + deletions: del === "-" ? null : Number(del), + }); + } + + const message = bodyRaw.replace(/\r\n/g, "\n").trimEnd(); + const nl = message.indexOf("\n"); + const subject = (nl === -1 ? message : message.slice(0, nl)).trim(); + const body = nl === -1 ? "" : message.slice(nl + 1).trim(); + + return { + sha: sha ?? "", + authorName: authorName ?? "", + authorEmail: authorEmail ?? "", + authorDate: authorDate ?? "", + commitDate: commitDate ?? "", + parents: (parentsRaw ?? "").split(" ").filter((p) => p.length > 0), + subject, + body, + files, + }; +} + +/** Options narrowing the `git log` walk. */ +export interface GitLogOptions { + /** Rev to walk (default HEAD). Ignored when `range` is set. */ + rev?: string; + /** Explicit rev range (e.g. `..HEAD`) for incremental walks. */ + range?: string; + /** Only commits at/after this date (passed to `git log --since`). */ + since?: string; + /** Only commits at/before this date (passed to `git log --until`). */ + until?: string; + /** Cap on walked commits (`git log --max-count`). */ + maxCount?: number; + /** Drop all merge commits in git itself (`git log --no-merges`). */ + noMerges?: boolean; +} + +/** + * Stream the commit log of the repo at `repoRoot`, newest first. + * + * An empty repo (no commits on the rev) yields nothing rather than failing, + * so `me claude init` is safe on fresh repos. + */ +export async function* walkGitLog( + repoRoot: string, + options: GitLogOptions = {}, +): AsyncIterable { + const args = [ + "-C", + repoRoot, + "log", + "--encoding=UTF-8", + "--numstat", + "--pretty=format:%x01%H%x00%an%x00%ae%x00%aI%x00%cI%x00%P%x00%B%x00", + ]; + if (options.since) args.push(`--since=${options.since}`); + if (options.until) args.push(`--until=${options.until}`); + if (options.maxCount !== undefined) + args.push(`--max-count=${options.maxCount}`); + if (options.noMerges) args.push("--no-merges"); + args.push(options.range ?? options.rev ?? "HEAD"); + args.push("--"); + + const proc = Bun.spawn(["git", ...args], { + stdout: "pipe", + stderr: "pipe", + }); + // Drain stderr concurrently so a chatty git can't fill the pipe and stall. + const stderrText = new Response(proc.stderr).text(); + + const parser = new GitLogParser(); + const decoder = new TextDecoder("utf-8"); + for await (const chunk of proc.stdout) { + yield* parser.push(decoder.decode(chunk, { stream: true })); + } + const final = decoder.decode(); + if (final.length > 0) yield* parser.push(final); + + const exitCode = await proc.exited; + if (exitCode !== 0) { + const stderr = await stderrText; + // A rev with no commits yet (fresh repo) is "nothing to import", not an + // error. Git phrases this a few ways depending on version/state. + if (/does not have any commits|unknown revision.*HEAD/i.test(stderr)) { + return; + } + throw new Error(`git log failed (exit ${exitCode}): ${stderr.trim()}`); + } + yield* parser.end(); +} + +/** + * True when `sha` is an ancestor of `rev` in the repo at `repoRoot`. Any + * failure (unknown sha after a force-push, detached state, …) is false — + * callers fall back to a full walk, which deterministic ids make safe. + */ +export async function isAncestor( + repoRoot: string, + sha: string, + rev: string, +): Promise { + try { + await execFileAsync( + "git", + ["-C", repoRoot, "merge-base", "--is-ancestor", sha, rev], + { timeout: 5000, encoding: "utf8" }, + ); + return true; + } catch { + return false; + } +} + +/** + * Why a merge commit is skipped, or null to import it. + * + * Default policy: keep merges that carry a message body (GitHub PR merges + * put the PR title there) and drop subject-only boilerplate + * (`Merge branch 'x' into y`). `--no-merges` drops them all — that case is + * filtered by git itself (see walkGitLog), so it never reaches here. + */ +export function mergeSkipReason(commit: GitCommit): string | null { + if (commit.parents.length < 2) return null; + return commit.body.length === 0 ? "merge_boilerplate" : null; +} + +/** Context shared by every memory built in one import run. */ +export interface CommitMemoryContext { + /** Full target tree (e.g. `share.projects.foo.git_history`). */ + tree: string; + /** Project slug the tree was derived from. */ + projectSlug: string; + /** Git remote URL, if the repo has one. */ + gitRemote?: string; + /** Render the changed-file list into the content. */ + fileList: boolean; + /** `meta.imported_at` for this run. */ + importedAt: string; +} + +/** + * Build the memory payload for one commit, or an error string when the + * commit has an unusable date. Content is the message plus a capped file + * list; everything queryable lives in meta; temporal is the commit date. + */ +export function buildCommitMemory( + commit: GitCommit, + ctx: CommitMemoryContext, +): MemoryCreateParams | { error: string } { + const commitMs = Date.parse(commit.commitDate); + if (Number.isNaN(commitMs)) { + return { error: `invalid commit date: ${commit.commitDate}` }; + } + + const id = deterministicUuidV7(`git:${ctx.tree}:${commit.sha}`, commitMs); + + let content = commit.subject; + const body = truncateUtf8(commit.body, BODY_BYTES_CAP); + if (body.length > 0) content += `\n\n${body}`; + if (ctx.fileList && commit.files.length > 0) { + const lines = commit.files + .slice(0, FILE_LIST_CAP) + .map((f) => + f.insertions === null || f.deletions === null + ? ` ${f.path} (binary)` + : ` ${f.path} (+${f.insertions} -${f.deletions})`, + ); + if (commit.files.length > FILE_LIST_CAP) { + lines.push(` … and ${commit.files.length - FILE_LIST_CAP} more files`); + } + content += `\n\nFiles:\n${lines.join("\n")}`; + } + + const insertions = commit.files.reduce( + (sum, f) => sum + (f.insertions ?? 0), + 0, + ); + const deletions = commit.files.reduce( + (sum, f) => sum + (f.deletions ?? 0), + 0, + ); + + const meta: Record = { + type: "git_commit", + sha: commit.sha, + source_project_slug: ctx.projectSlug, + author_name: commit.authorName, + author_email: commit.authorEmail, + author_date: commit.authorDate, + commit_date: commit.commitDate, + files_changed: commit.files.length, + insertions, + deletions, + imported_at: ctx.importedAt, + importer_version: GIT_IMPORTER_VERSION, + }; + if (ctx.gitRemote) meta.source_git_repo = ctx.gitRemote; + if (commit.parents.length >= 2) meta.is_merge = true; + + return { + id, + content, + meta, + tree: ctx.tree, + temporal: { start: new Date(commitMs).toISOString() }, + }; +} + +/** Truncate to a UTF-8 byte budget on a char boundary, marking the cut. */ +function truncateUtf8(text: string, maxBytes: number): string { + if (Buffer.byteLength(text, "utf8") <= maxBytes) return text; + let end = text.length; + while (end > 0 && Buffer.byteLength(text.slice(0, end), "utf8") > maxBytes) { + // Shrink geometrically instead of stepping one char at a time. + end = Math.min(end - 1, Math.floor(end * 0.9)); + } + // Don't cut a surrogate pair in half. + const last = text.charCodeAt(end - 1); + if (last >= 0xd800 && last <= 0xdbff) end--; + return `${text.slice(0, end)}\n…[truncated]`; +} diff --git a/packages/cli/importers/import-transcript.test.ts b/packages/cli/importers/import-transcript.test.ts index e0720f8..0fea71a 100644 --- a/packages/cli/importers/import-transcript.test.ts +++ b/packages/cli/importers/import-transcript.test.ts @@ -72,7 +72,7 @@ function importerFor(session: ImportedSession | null): Importer { }; } -/** An importer whose discoverSessions yields one fixed session (the `me import` path). */ +/** An importer whose discoverSessions yields one fixed session (the `me import claude` path). */ function discoverImporter(session: ImportedSession): Importer { return { tool: "claude", @@ -158,11 +158,11 @@ describe("importTranscriptFile", () => { expect(store.size).toBe(4); }); - // The hook (importTranscriptFile) and `me import` (runImport) must be + // The hook (importTranscriptFile) and `me import claude` (runImport) must be // idempotent w.r.t. each other: both derive the same tree + deterministic ids // from the same parse, so importing a session via one path and then the other // inserts nothing the second time. Guards the shared-derivation assumption. - test("hook capture then `me import` over the same session is a no-op", async () => { + test("hook capture then `me import claude` over the same session is a no-op", async () => { const { client, store } = mockEngine(); const s = session(["a", "b", "c"]); @@ -180,7 +180,7 @@ describe("importTranscriptFile", () => { expect(store.size).toBe(3); }); - test("`me import` then hook capture over the same session is a no-op", async () => { + test("`me import claude` then hook capture over the same session is a no-op", async () => { const { client, store } = mockEngine(); const s = session(["a", "b", "c"]); diff --git a/packages/cli/importers/index.ts b/packages/cli/importers/index.ts index 3b8da99..f007a6a 100644 --- a/packages/cli/importers/index.ts +++ b/packages/cli/importers/index.ts @@ -47,7 +47,7 @@ export const IMPORTER_VERSION = "1"; const SEARCH_PAGE_LIMIT = 1000; /** - * Default capture layout, shared by `me import` and the Claude Code capture + * Default capture layout, shared by `me import claude` and the Claude Code capture * hook so live + imported sessions land in the same place: * `..`. Under * `share` so a session-authenticated user (owner@share, not arbitrary top-level @@ -180,7 +180,7 @@ export async function runImport( /** * Import a single transcript file — the live-capture path used by the Claude - * Code hook. Reuses the same parse + render + write as `me import`, so live + * Code hook. Reuses the same parse + render + write as `me import claude`, so live * captures and bulk imports produce identical memories (tree, ids, `source_*` * metadata). * @@ -246,11 +246,12 @@ export async function importTranscriptFile( delta.map((d) => d.payload), ); if (errors.length === 0) { + // An already-present id (non-monotonic transcript) is silently + // skipped server-side, so inserted may be < delta.length. outcome.inserted += insertedIds.length; return outcome; } - // Partial failure (e.g. an already-present id from a non-monotonic - // transcript) → fall through to the full reconcile for correctness. + // A failed chunk → fall through to the full reconcile for correctness. } catch { // Unexpected write error → reconcile. } diff --git a/packages/cli/importers/uuid.test.ts b/packages/cli/importers/uuid.test.ts index 8c9a4ab..5a3b126 100644 --- a/packages/cli/importers/uuid.test.ts +++ b/packages/cli/importers/uuid.test.ts @@ -1,8 +1,8 @@ /** - * Tests for deterministic per-message UUIDv7 derivation. + * Tests for deterministic UUIDv7 derivation. */ import { describe, expect, test } from "bun:test"; -import { deterministicMessageUuidV7 } from "./uuid.ts"; +import { deterministicMessageUuidV7, deterministicUuidV7 } from "./uuid.ts"; const UUIDV7_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -103,4 +103,23 @@ describe("deterministicMessageUuidV7", () => { // Position 19 = variant high nibble; top 2 bits must be 10 → hex 8/9/a/b. expect(["8", "9", "a", "b"]).toContain(id.charAt(19)); }); + + test("equals deterministicUuidV7 over the tool:session:message key", () => { + // The message variant is a thin wrapper; the key format is load-bearing + // for ids already written to engines, so lock it down. + const ts = 1_700_000_000_000; + expect(deterministicMessageUuidV7("claude", "abc", "m1", ts)).toBe( + deterministicUuidV7("claude:abc:m1", ts), + ); + }); +}); + +describe("deterministicUuidV7", () => { + test("namespaced keys produce distinct ids at the same timestamp", () => { + const ts = 1_700_000_000_000; + const a = deterministicUuidV7("git:share.projects.x.git_history:abc", ts); + const b = deterministicUuidV7("git:share.projects.y.git_history:abc", ts); + expect(a).toMatch(UUIDV7_RE); + expect(a).not.toBe(b); + }); }); diff --git a/packages/cli/importers/uuid.ts b/packages/cli/importers/uuid.ts index 19323d9..2077921 100644 --- a/packages/cli/importers/uuid.ts +++ b/packages/cli/importers/uuid.ts @@ -1,35 +1,32 @@ /** * Deterministic UUIDv7 derivation for idempotent imports. * - * We need stable UUIDs so that re-importing the same message collides + * We need stable UUIDs so that re-importing the same record collides * with the existing row in the database and becomes a no-op. Regular * UUIDv7 is random, so we derive a deterministic variant: * - * - 48 bits: Unix ms timestamp (message timestamp) — keeps chronological sort + * - 48 bits: Unix ms timestamp (record timestamp) — keeps chronological sort * - 4 bits: version = 7 - * - 12 bits: rand_a ← SHA-256(tool + ':' + sessionId + ':' + messageId), bits 0..11 + * - 12 bits: rand_a ← SHA-256(key), bits 0..11 * - 2 bits: variant = 10 - * - 62 bits: rand_b ← SHA-256(tool + ':' + sessionId + ':' + messageId), bits 12..73 + * - 62 bits: rand_b ← SHA-256(key), bits 12..73 * * The result passes the `uuid_extract_version(id) = 7` check in the engine's - * memory schema, sorts by message time, and is stable across re-imports of + * memory schema, sorts by record time, and is stable across re-imports of * the same source data. */ import { createHash } from "node:crypto"; import type { SourceTool } from "./types.ts"; /** - * Compute a deterministic UUIDv7 from `(tool, sessionId, messageId, timestampMs)`. + * Compute a deterministic UUIDv7 from an identity `key` and a timestamp. * * Same inputs always return the same UUID; different inputs produce - * different UUIDs (cryptographically, with SHA-256). + * different UUIDs (cryptographically, with SHA-256). Each importer owns + * its key format (messages: `tool:sessionId:messageId`; git commits: + * `git::`) — keys must be namespaced so importers can't collide. */ -export function deterministicMessageUuidV7( - tool: SourceTool, - sessionId: string, - messageId: string, - timestampMs: number, -): string { +export function deterministicUuidV7(key: string, timestampMs: number): string { // 16 bytes = 128 bits. const bytes = new Uint8Array(16); @@ -42,11 +39,9 @@ export function deterministicMessageUuidV7( bytes[4] = Math.floor(ts / 2 ** 8) & 0xff; bytes[5] = ts & 0xff; - // SHA-256 over "tool:sessionId:messageId" gives 32 bytes of deterministic - // pseudo-random. We only need 74 bits (12 + 62) so 10 bytes is plenty. - const digest = createHash("sha256") - .update(`${tool}:${sessionId}:${messageId}`, "utf8") - .digest(); + // SHA-256 over the key gives 32 bytes of deterministic pseudo-random. + // We only need 74 bits (12 + 62) so 10 bytes is plenty. + const digest = createHash("sha256").update(key, "utf8").digest(); // Bytes 6..7: version (4 bits = 0x7) + rand_a (12 bits). const randA = ((digest[0] ?? 0) << 8) | (digest[1] ?? 0); @@ -63,6 +58,19 @@ export function deterministicMessageUuidV7( return bytesToUuid(bytes); } +/** + * Compute a deterministic UUIDv7 from `(tool, sessionId, messageId, timestampMs)`. + * The message-import key format; see `deterministicUuidV7`. + */ +export function deterministicMessageUuidV7( + tool: SourceTool, + sessionId: string, + messageId: string, + timestampMs: number, +): string { + return deterministicUuidV7(`${tool}:${sessionId}:${messageId}`, timestampMs); +} + /** Format 16 bytes as a canonical UUID string. */ function bytesToUuid(bytes: Uint8Array): string { const hex: string[] = []; diff --git a/packages/cli/index.ts b/packages/cli/index.ts index 7db4219..d157a1a 100755 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -16,6 +16,7 @@ import { createClaudeCommand } from "./commands/claude.ts"; import { createCodexCommand } from "./commands/codex.ts"; import { createGeminiCommand } from "./commands/gemini.ts"; import { createGroupCommand } from "./commands/group.ts"; +import { createImportCommand } from "./commands/import-group.ts"; import { createLoginCommand } from "./commands/login.ts"; import { createLogoutCommand } from "./commands/logout.ts"; import { createMcpCommand } from "./commands/mcp.ts"; @@ -75,6 +76,9 @@ program.addCommand(createApiKeyCommand()); program.addCommand(createMemoryCommand()); for (const c of createMemoryAliasCommands()) program.addCommand(c); +// Import group — one subcommand per source (`me import memories|claude|codex|opencode|git`) +program.addCommand(createImportCommand()); + // MCP server program.addCommand(createMcpCommand()); diff --git a/packages/docs-site/lib/nav.ts b/packages/docs-site/lib/nav.ts index 51d8593..d2a3d9a 100644 --- a/packages/docs-site/lib/nav.ts +++ b/packages/docs-site/lib/nav.ts @@ -34,6 +34,7 @@ export const NAV: NavSection[] = [ { label: "me whoami", slug: "cli/me-whoami" }, { label: "me space", slug: "cli/me-space" }, { label: "me memory", slug: "cli/me-memory" }, + { label: "me import", slug: "cli/me-import" }, { label: "me mcp", slug: "cli/me-mcp" }, { label: "me claude", slug: "cli/me-claude" }, { label: "me codex", slug: "cli/me-codex" }, diff --git a/scripts/integration-test.ts b/scripts/integration-test.ts index 47fc159..1f38cfa 100644 --- a/scripts/integration-test.ts +++ b/scripts/integration-test.ts @@ -1053,6 +1053,18 @@ async function phase5_memory(): Promise { expectEq(json.failed, 0, "failed count"); }); + // Canonical spelling of the same command (`me memory import` is its alias). + await step("me import memories (--dry-run)", async () => { + const { json } = await runJson<{ wouldImport: number; dryRun: boolean }>([ + "import", + "memories", + join(fixturesDir, "sample.md"), + "--dry-run", + ]); + expectEq(json.dryRun, true, "dryRun"); + expectEq(json.wouldImport, 1, "wouldImport"); + }); + await step("me memory import (recursive --dry-run)", async () => { // exclude the pack yaml from the picture by importing only the // sample files via the directory recursion. From fc23440ae1226995b9c039ad0b328c9a880e53d4 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 16:54:26 +0200 Subject: [PATCH 135/156] feat(server): re-migrate all existing space schemas at boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spaces were migrated only once, at provision time — so a deploy that changed the idempotent space SQL (the function bodies in space/migrate/idempotent/*.sql) never reached existing spaces: boot migrated core + auth only, and migrateSpace (built as the standalone re-migrate path) had no callers in the server. The on-conflict fix in create_memory, for example, was undeployable to any space that already existed. startServer now enumerates core.space after the core/auth migrations and runs migrateSpace for each (mirroring provisionSpace's defaults). Re-running is cheap — incrementals are version-tracked no-ops, idempotent files are re-applied — and concurrent replica boots are serialized by the per-schema advisory lock. Every space is attempted so the logs name each broken one, then any failure aborts boot. The boot integration test now provisions a space whose create_memory has been tampered with before startServer runs and asserts the sweep restores the real definition. Also rewrites DEVELOPMENT.md's stale "Adding a migration" section (retired engine/accounts paths) to describe the current auth/core/space layout and the boot-time sweep. Co-Authored-By: Claude Fable 5 --- DEVELOPMENT.md | 25 ++++--- packages/server/start.integration.test.ts | 80 ++++++++++++++++++++++- packages/server/start.ts | 43 +++++++++++- 3 files changed, 133 insertions(+), 15 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f1eb368..2abb097 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -381,16 +381,21 @@ Implications: ### Adding a migration -Database migrations run at server startup using `SERVER_VERSION` as the -`serverVersion` passed to the runners. When you add a new migration file under -`packages/engine/migrate/migrations/` or `packages/accounts/migrate/migrations/`: - -- Cut a **server** release afterwards (`./bun run release:server`) to advance - `SERVER_VERSION` and propagate the migration to prod. -- The migration's `applied_at_version` column and the schema's `version` - row will reflect the new `SERVER_VERSION`. -- Rolling back to an older server image will then trip the downgrade guard - in the migration runners — by design. +Migrations live under `packages/database/{auth,core,space}/migrate/` in two +flavors: `incremental/` (versioned DDL, applied exactly once per schema, +tracked by name) and `idempotent/` (function/index definitions, re-applied on +every migrate run via `create or replace`). + +Migrations run at server startup (`startServer`, unless `migrate: false`): +the `auth` and `core` schemas are migrated, then **every existing space +schema is re-migrated** (enumerated from `core.space`) so changes to the +idempotent space SQL reach already-provisioned spaces on the next deploy — +spaces are otherwise only migrated once, when provisioned. A failed space +re-migration aborts boot; concurrent replica boots are serialized by a +per-schema advisory lock. + +Rolling back to an older server image trips the downgrade guard in the +migration runners (stamped schema version newer than the app's) — by design. ## Troubleshooting diff --git a/packages/server/start.integration.test.ts b/packages/server/start.integration.test.ts index d2924ba..bed8002 100644 --- a/packages/server/start.integration.test.ts +++ b/packages/server/start.integration.test.ts @@ -3,13 +3,24 @@ // Boots the real server stack (pools → migrate → worker → Bun.serve) against // isolated auth/core test schemas on a port-0 listener, then hits /health and // /ready. No real embeddings are exercised (a placeholder key suffices — the -// worker idles with no spaces), so this needs no OpenAI key. +// worker idles), so this needs no OpenAI key. // TEST_DATABASE_URL="$(ghost connect testing_me)" \ // bun test --timeout 30000 packages/server/start.integration.test.ts + +// Space schemas created by this test land under metest_ (not production +// me_) so leftovers are reclaimable by name. Set before anything reads it. +process.env.SPACE_SCHEMA_PREFIX = "metest_"; + import { afterAll, beforeAll, expect, test } from "bun:test"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; import type { EmbeddingConfig } from "@memory.build/embedding"; import postgres, { type Sql } from "postgres"; import { startServer } from "./lib"; +import { provisionUser } from "./provision"; const URL = process.env.TEST_DATABASE_URL ?? @@ -35,10 +46,62 @@ let sql: Sql; let srv: Awaited>; let authSchema: string; let coreSchema: string; +let spaceSchema: string; +let tamperedDef: string; +let bootedDef: string; + +/** Current definition of a space schema's create_memory function. */ +async function createMemoryDef(schema: string): Promise { + const [row] = await sql` + select pg_get_functiondef(p.oid) as def + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = ${schema} and p.proname = 'create_memory'`; + return (row?.def as string) ?? ""; +} beforeAll(async () => { authSchema = `auth_test_${rand()}`; coreSchema = `core_test_${rand()}`; + sql = postgres(URL, { onnotice: () => {} }); + + // Simulate an existing deployment: a fully provisioned space whose + // create_memory predates the current idempotent SQL. Migrate the control + // plane, provision a user (+ its default space), then tamper the space's + // create_memory so we can prove boot re-applies the real definition. + await bootstrapSpaceDatabase(sql); + await migrateCore(sql, { schema: coreSchema }); + await migrateAuth(sql, { schema: authSchema }); + const provisioned = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: "boot@example.test", + name: "Boot", + provider: "github", + accountId: `boot-${rand()}`, + emailVerified: true, + }, + ); + spaceSchema = `metest_${provisioned.spaceSlug}`; + await sql.unsafe(` + create or replace function ${spaceSchema}.create_memory + ( _tree_access jsonb + , _tree ltree + , _content text + , _id uuid default null + , _meta jsonb default '{}' + , _temporal tstzrange default null + ) + returns uuid + as $func$ + begin + return null; -- stale stand-in, must be replaced by the boot sweep + end; + $func$ language plpgsql volatile + `); + tamperedDef = await createMemoryDef(spaceSchema); + srv = await startServer({ port: 0, databaseUrl: URL, @@ -50,13 +113,15 @@ beforeAll(async () => { workerIdleDelayMs: 250, workerRefreshIntervalMs: 500, enableCleanupCron: false, - // migrate defaults to true — startServer migrates the isolated schemas. + // migrate defaults to true — startServer migrates the isolated schemas + // and re-migrates the pre-existing space. }); - sql = postgres(URL, { onnotice: () => {} }); + bootedDef = await createMemoryDef(spaceSchema); }); afterAll(async () => { await srv?.stop(); + await sql.unsafe(`drop schema if exists ${spaceSchema} cascade`); await sql.unsafe(`drop schema if exists ${authSchema} cascade`); await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); await sql.end(); @@ -86,3 +151,12 @@ test("migrated the configured isolated schemas", async () => { expect(Boolean(authRow?.e)).toBe(true); expect(Boolean(coreRow?.e)).toBe(true); }); + +test("re-migrates existing space schemas on boot", async () => { + // The tampered function was in place just before boot… + expect(tamperedDef).toContain("stale stand-in"); + expect(tamperedDef).not.toContain("on conflict"); + // …and boot's space sweep re-applied the idempotent SQL over it. + expect(bootedDef).not.toContain("stale stand-in"); + expect(bootedDef).toContain("on conflict (id) do nothing"); +}); diff --git a/packages/server/start.ts b/packages/server/start.ts index 268bd93..c5977b2 100644 --- a/packages/server/start.ts +++ b/packages/server/start.ts @@ -12,17 +12,18 @@ import { bootstrapSpaceDatabase, migrateAuth, migrateCore, + migrateSpace, slugToSchema as spaceSlugToSchema, } from "@memory.build/database"; import type { EmbeddingConfig } from "@memory.build/embedding"; -import { coreStore } from "@memory.build/engine/core"; +import { type CoreStore, coreStore } from "@memory.build/engine/core"; import { DEFAULT_WORKER_TIMEOUTS, WorkerPool, type WorkerTimeouts, } from "@memory.build/worker"; import { info, reportError, span } from "@pydantic/logfire-node"; -import postgres from "postgres"; +import postgres, { type Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import { embeddingConstants } from "./config"; import type { ServerContext } from "./context"; @@ -129,6 +130,43 @@ export interface RunningServer { stop(): Promise; } +/** + * Re-migrate every existing space schema at boot. + * + * Spaces are otherwise migrated only once, at provision time — so a deploy + * that changes the idempotent space SQL (the function bodies in + * space/migrate/idempotent/*.sql) would never reach existing spaces without + * this sweep. Re-running is cheap: incremental migrations are version-tracked + * no-ops, idempotent files are re-applied (create or replace). Options mirror + * provisionSpace (all defaults), and migrateSpace's per-schema advisory lock + * serializes concurrent replica boots. + * + * Every space is attempted (so one broken space doesn't hide the rest from + * the logs), then any failure aborts boot — the server must not serve spaces + * whose schema may be stale. + */ +async function remigrateSpaces(db: Sql, core: CoreStore): Promise { + const spaces = await core.listSpaces(); + const failed: string[] = []; + for (const space of spaces) { + try { + await migrateSpace(db, { slug: space.slug }); + } catch (error) { + failed.push(space.slug); + reportError( + `Space ${space.slug} re-migration failed`, + error instanceof Error ? error : new Error(String(error)), + ); + } + } + if (failed.length > 0) { + throw new Error( + `space re-migration failed for ${failed.length} of ${spaces.length} space(s): ${failed.join(", ")}`, + ); + } + info(`${spaces.length} space schema(s) re-migrated`); +} + /** * Boot the server stack and return a handle. No process-level side effects — * the caller owns signal handling and process exit (index.ts does this). @@ -290,6 +328,7 @@ export async function startServer( await migrateCore(db, { schema: coreSchema }); await migrateAuth(db, { schema: authSchema }); info("Core + auth schemas migrated"); + await remigrateSpaces(db, core); } // --------------------------------------------------------------------------- From 91435fca885c2555ae2c45558dd49beafd6b74b9 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 18:59:02 +0200 Subject: [PATCH 136/156] feat(server): conditional upsert in create_memory; drop the importer's capped lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_memory gains `_replace_if_meta_differs text` and now does insert / update-if-stale / skip atomically: on a duplicate explicit id it replaces the row (tree/meta/temporal/content) when the stored meta value for the caller-named key differs from the new record's, else skips. Returns (id, inserted) — xmax = 0 distinguishes a fresh insert from a replace; zero rows = skipped. The replace arm additionally requires write access on the existing row's tree (silently skipped without it, so one inaccessible row can't fail a batch). The old 6-arg signature is dropped first — a new defaulted arg would create an ambiguous overload, and the return type changed; the boot-time space sweep delivers the new function to existing spaces. Embedding columns are untouched: the content-gated update triggers mean a meta-only replace neither invalidates nor re-enqueues the embedding. memory.batchCreate threads the key (`replaceIfMetaDiffers`) and returns `{ids, updatedIds}`; the session importers pass "importer_version", so a version bump re-renders previously-imported messages server-side, batched. That deletes the client-side reconcile in the importers: writeSession's existing-state pre-fetch (`fetchExistingMessageVersions`) was one search capped at 1000 rows — sessions past the cap silently re-submitted their older messages every import with vanishing counts, and the guard meant to catch it was dead code (search `total` is the page length). Now every planned message goes through the upsert and inserted/updated/skipped come from the batch response, exact at any session size; the hook keeps its high-water search purely as a bandwidth optimization. Also raises the CLI's memory-client timeout to 120s: 1000-row chunks are processed row-by-row server-side and can legitimately exceed 30s. Verified end-to-end against a real 27-session backfill including a >1000-message session: 3269 inserted + 11984 cross-file duplicates skipped, zero failures; re-import all-skip with honest counts. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 1 + docs/cli/agent-session-imports.md | 8 +- docs/cli/me-memory.md | 2 +- e2e/cli.e2e.test.ts | 76 ++++- packages/cli/chunk.test.ts | 62 +++- packages/cli/chunk.ts | 42 ++- .../cli/importers/import-transcript.test.ts | 66 ++++- packages/cli/importers/index.ts | 277 ++++++------------ packages/cli/util.ts | 4 + .../space/migrate/idempotent/001_memory.sql | 45 ++- .../space/migrate/migrate.integration.test.ts | 128 +++++++- packages/engine/space/db.integration.test.ts | 45 ++- packages/engine/space/db.ts | 25 +- packages/engine/space/types.ts | 6 + packages/protocol/memory.ts | 13 + .../rpc/memory/memory.integration.test.ts | 78 ++++- packages/server/rpc/memory/memory.ts | 33 ++- packages/server/start.integration.test.ts | 2 +- 18 files changed, 634 insertions(+), 279 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 38a094b..15f2c5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -178,6 +178,7 @@ create table me.memory - **CLI credentials**: split across `~/.config/me/` — **`config.yaml`** (non-secret: default server + per-server **active space** / the X-Me-Space) and **`credentials.yaml`** (0600, secret session-token *fallback* only). The **session token** lives in the OS keychain when available (macOS `security`, Linux `secret-tool` via libsecret; `ME_NO_KEYCHAIN=1` forces off), else in `credentials.yaml` (empty/absent on keychain hosts); a pre-split `credentials.yaml` is migrated on first read. `me logout` clears the session secret but keeps the non-secret config (so re-login resumes). **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE` / `ME_SESSION_TOKEN` / `ME_NO_KEYCHAIN`. - **Header constants** (`CLIENT_VERSION_HEADER`, `SPACE_HEADER`) live in `@memory.build/protocol/headers`. - **MCP compatibility**: all tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). +- **batchCreate conflict semantics**: a duplicate explicit id is skipped, or — with `replaceIfMetaDiffers: ""` — replaced in place when the stored row's value for that key differs (the session importers pass `importer_version` so version bumps re-render server-side). Result is `{ids, updatedIds}` (inserted / replaced); ids in neither were skipped. Single `memory.create` on a duplicate id errors with CONFLICT. ## Database driver: postgres.js diff --git a/docs/cli/agent-session-imports.md b/docs/cli/agent-session-imports.md index 1be2961..bb46646 100644 --- a/docs/cli/agent-session-imports.md +++ b/docs/cli/agent-session-imports.md @@ -46,13 +46,11 @@ Project slugs come from the git repo root directory name when the cwd is inside ## Idempotency -Each imported message gets a deterministic UUIDv7 derived from `(tool, session_id, message_id, timestamp)`. On re-import: +Each imported message gets a deterministic UUIDv7 derived from `(tool, session_id, message_id, timestamp)`. Re-imports reconcile **server-side**: every planned message is submitted through the engine's conditional upsert, which inserts new ids, rewrites in place any row whose stored `meta.importer_version` differs from the current importer's (so a version bump re-renders previously-imported messages in the same batched pass), and skips rows that are already current. There is no per-session lookup and no session-size limit — a session with tens of thousands of imported messages reconciles exactly like a small one. -1. The importer looks up each message by that id. -2. If the memory already exists and `meta.importer_version` matches, it is skipped. -3. Otherwise the memory is (re)written. +Source files are append-only for all three tools, so re-importing an in-progress session simply inserts its newly-appended messages on the next run. The live-capture hook additionally narrows each submission to the messages after the newest already-imported one (a single `limit 1` search) — purely a bandwidth optimization; correctness never depends on it. -Source files are append-only for all three tools, so re-importing an in-progress session simply inserts its newly-appended messages on the next run. +`--dry-run` reports every parsed message as a would-be insert: without submitting, there is no server classification into inserted/updated/skipped. ## Content shape diff --git a/docs/cli/me-memory.md b/docs/cli/me-memory.md index 75ba084..c469813 100644 --- a/docs/cli/me-memory.md +++ b/docs/cli/me-memory.md @@ -235,7 +235,7 @@ Supports Markdown (with YAML frontmatter), YAML, JSON, and NDJSON. Format is aut ### Skipped memories -Memories with an explicit `id` that already exists in the space are silently skipped server-side (via `ON CONFLICT DO NOTHING`) rather than failing the whole batch. The command surfaces these as `skipped` so re-imports of unchanged data and id collisions with unrelated memories are observable. Memories without an `id` get a server-generated UUIDv7 and never collide. +Memories with an explicit `id` that already exists in the space are silently skipped server-side (a conflict skip in `create_memory`) rather than failing the whole batch. The command surfaces these as `skipped` so re-imports of unchanged data and id collisions with unrelated memories are observable. Memories without an `id` get a server-generated UUIDv7 and never collide. JSON output adds `skipped` (count) and `skippedIds` (array of conflicting ids). Text output appends `(K skipped — id already exists)` to the summary, or prints `Imported 0 memories (N already exist, no changes)` when everything was a re-import. Run with `--verbose` to see each skipped id inline. diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index b01bda9..f9203a6 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -815,7 +815,81 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( await rm(root, { recursive: true, force: true }); }); - test("9b. `me import` group: no bare default, memories ≡ memory import", async () => { + test("9b. a stale importer_version is re-rendered in place on re-import", async () => { + // The server's conditional upsert: re-importing a session rewrites any + // row whose stored meta.importer_version differs from the current + // importer's, and skips the rest — no client-side existing-state read. + const sessionId = `stale-${rand()}`; + const root = await mkdtemp(join(tmpdir(), "me-e2e-stale-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-05-01T00:00:0${i}.000Z`, + sessionId, + cwd: "/work/stale-proj", + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + await writeFile( + join(projDir, `${sessionId}.jsonl`), + [ + mkMsg(0, "user", "stale first question"), + mkMsg(1, "assistant", "stale first answer"), + mkMsg(2, "user", "stale second question"), + mkMsg(3, "assistant", "stale second answer"), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + + const first = await meJson<{ inserted: number }>([ + "import", + "claude", + "--source", + root, + ]); + expect(first.inserted).toBe(4); + + // Rewind one row to look like an older importer build wrote it. + const [stale] = await sql.unsafe( + `update metest_${spaceSlug}.memory + set content = 'STALE RENDER', + meta = jsonb_set(meta, '{importer_version}', '"0"') + where meta->>'source_session_id' = $1 + and meta->>'source_message_id' = $2 + returning id`, + [sessionId, `${sessionId}-user-0`], + ); + expect(stale?.id).toBeDefined(); + + // Re-import: exactly the stale row is rewritten, the rest skip. + const second = await meJson<{ + inserted: number; + updated: number; + skipped: number; + failed: number; + }>(["import", "claude", "--source", root]); + expect(second.inserted).toBe(0); + expect(second.updated).toBe(1); + expect(second.skipped).toBe(3); + expect(second.failed).toBe(0); + + const [row] = await sql.unsafe( + `select content, meta->>'importer_version' as v + from metest_${spaceSlug}.memory where id = $1`, + [stale?.id as string], + ); + expect(row?.content).toBe("stale first question"); + expect(row?.v).toBe("1"); + + await rm(root, { recursive: true, force: true }); + }); + + test("9c. `me import` group: no bare default, memories ≡ memory import", async () => { // Bare `me import` is a group, not the old file-import alias: it prints // the subcommand list and exits non-zero. const bare = await me(["import"]); diff --git a/packages/cli/chunk.test.ts b/packages/cli/chunk.test.ts index 5e62393..f916944 100644 --- a/packages/cli/chunk.test.ts +++ b/packages/cli/chunk.test.ts @@ -111,9 +111,19 @@ describe("batchCreateChunked", () => { /** Minimal stub client; the test supplies the per-call behavior. */ const stubClient = ( - handler: (memories: MemoryCreateParams[]) => Promise<{ ids: string[] }>, + handler: ( + memories: MemoryCreateParams[], + replaceIfMetaDiffers?: string, + ) => Promise<{ ids: string[]; updatedIds?: string[] }>, ): BatchCreateClient => ({ - memory: { batchCreate: ({ memories }) => handler(memories) }, + memory: { + batchCreate: async ({ memories, replaceIfMetaDiffers }) => { + const res = await handler(memories, replaceIfMetaDiffers); + // Old servers omit updatedIds; the helper must tolerate that, so the + // stub passes whatever the handler chose to return. + return res as { ids: string[]; updatedIds: string[] }; + }, + }, }); test("single chunk, all succeed", async () => { @@ -183,12 +193,13 @@ describe("batchCreateChunked", () => { }); test("server returns shorter ids than requested (simulating ON CONFLICT)", async () => { - // Mimics post-#64 server behavior: caller submits 3 memories, server - // inserts 2 (one was a duplicate id, dropped by ON CONFLICT). The - // helper should faithfully report the 2 inserted; classifying the - // missing one as "skipped" is the caller's job. + // Caller submits 3 memories, server inserts 2 (one was a duplicate id, + // skipped by the conditional upsert). The helper should faithfully + // report the 2 inserted; classifying the missing one as "skipped" is + // the caller's job. const client = stubClient(async (memories) => ({ ids: memories.map((m) => m.id ?? "auto").filter((id) => id !== "dup"), // server "drops" the dup id + updatedIds: [], })); const result = await batchCreateChunked(client, [ mem("a"), @@ -196,18 +207,55 @@ describe("batchCreateChunked", () => { mem("b"), ]); expect(result.insertedIds).toEqual(["a", "b"]); + expect(result.updatedIds).toEqual([]); expect(result.failedIds).toEqual([]); // no chunk failed expect(result.errors).toEqual([]); }); + test("passes replaceIfMetaDiffers through and accumulates updatedIds", async () => { + // Two chunks (big payloads); the server reports the first id of each + // chunk as updated and the rest as inserted. + const seenKeys: Array = []; + const client = stubClient(async (memories, replaceIfMetaDiffers) => { + seenKeys.push(replaceIfMetaDiffers); + const ids = memories.map((m) => m.id ?? "auto"); + return { ids: ids.slice(1), updatedIds: ids.slice(0, 1) }; + }); + const result = await batchCreateChunked( + client, + [mem("a", 700_000), mem("b", 10), mem("c", 700_000), mem("d", 10)], + { replaceIfMetaDiffers: "importer_version" }, + ); + expect(seenKeys.length).toBeGreaterThan(1); // multiple chunks + expect(new Set(seenKeys)).toEqual(new Set(["importer_version"])); + expect(result.updatedIds.length).toBe(seenKeys.length); + expect([...result.insertedIds, ...result.updatedIds].sort()).toEqual([ + "a", + "b", + "c", + "d", + ]); + }); + + test("tolerates a pre-upsert server omitting updatedIds", async () => { + const client = stubClient(async (memories) => ({ + ids: memories.map((m) => m.id ?? "auto"), + // no updatedIds field at all + })); + const result = await batchCreateChunked(client, [mem("a")]); + expect(result.insertedIds).toEqual(["a"]); + expect(result.updatedIds).toEqual([]); + }); + test("empty input never calls the server", async () => { let calls = 0; const client = stubClient(async () => { calls++; - return { ids: [] }; + return { ids: [], updatedIds: [] }; }); const result = await batchCreateChunked(client, []); expect(result.insertedIds).toEqual([]); + expect(result.updatedIds).toEqual([]); expect(result.failedIds).toEqual([]); expect(result.errors).toEqual([]); expect(calls).toBe(0); diff --git a/packages/cli/chunk.ts b/packages/cli/chunk.ts index df7e05f..f65069c 100644 --- a/packages/cli/chunk.ts +++ b/packages/cli/chunk.ts @@ -115,14 +115,28 @@ export interface BatchCreateClient { memory: { batchCreate: (params: { memories: MemoryCreateParams[]; - }) => Promise<{ ids: string[] }>; + replaceIfMetaDiffers?: string; + }) => Promise<{ ids: string[]; updatedIds: string[] }>; }; } +/** Options applied to every chunk of a `batchCreateChunked` run. */ +export interface BatchCreateChunkedOptions { + /** + * Meta key for the server's conditional replace: a memory whose explicit + * id already exists is rewritten in place when the stored row's value for + * this key differs (importers pass "importer_version" so version bumps + * re-render existing rows), else skipped. Unset: duplicates are skipped. + */ + replaceIfMetaDiffers?: string; +} + /** Result of a chunked `batchCreate` run. */ export interface BatchCreateChunkedResult { /** Ids the server confirmed inserted (across all successful chunks). */ insertedIds: string[]; + /** Existing rows rewritten in place via `replaceIfMetaDiffers`. */ + updatedIds: string[]; /** * Explicit ids submitted in chunks that errored, flattened across all * failed chunks for callers that just need a set of "ids to exclude @@ -151,26 +165,36 @@ export interface BatchCreateChunkedResult { * * Chunks are sent sequentially. A failed chunk is recorded once in * `errors` and its explicit ids are added to `failedIds`; it does not - * abort siblings. Successful chunks contribute to `insertedIds`. + * abort siblings. Successful chunks contribute to `insertedIds` and + * `updatedIds`. * - * Note: the returned `insertedIds` may be shorter than the number of - * inputs in successful chunks because the server uses - * `ON CONFLICT (id) DO NOTHING`. Use `computeSkippedIds` (or, for packs, - * `classifySkips` with `failedIds`) to classify the missing ids. + * A submitted explicit id in neither array (and not in a failed chunk) was + * skipped server-side — it already exists, at a matching meta-key value + * when `replaceIfMetaDiffers` is set. Use `computeSkippedIds` (or, for + * packs, `classifySkips` with `failedIds`) to classify the missing ids. */ export async function batchCreateChunked( client: BatchCreateClient, memories: MemoryCreateParams[], + options: BatchCreateChunkedOptions = {}, ): Promise { const insertedIds: string[] = []; + const updatedIds: string[] = []; const failedIds: string[] = []; const errors: BatchCreateChunkedResult["errors"] = []; let chunkIndex = 0; for (const chunk of chunkMemoriesForBatchCreate(memories)) { try { - const { ids } = await client.memory.batchCreate({ memories: chunk }); - insertedIds.push(...ids); + const res = await client.memory.batchCreate({ + memories: chunk, + ...(options.replaceIfMetaDiffers !== undefined + ? { replaceIfMetaDiffers: options.replaceIfMetaDiffers } + : {}), + }); + insertedIds.push(...res.ids); + // A pre-upsert server doesn't return updatedIds; treat as none updated. + updatedIds.push(...(res.updatedIds ?? [])); } catch (error) { const msg = error instanceof Error ? error.message : String(error); const ids = chunk @@ -187,5 +211,5 @@ export async function batchCreateChunked( chunkIndex++; } - return { insertedIds, failedIds, errors }; + return { insertedIds, updatedIds, failedIds, errors }; } diff --git a/packages/cli/importers/import-transcript.test.ts b/packages/cli/importers/import-transcript.test.ts index 0fea71a..7e96a9d 100644 --- a/packages/cli/importers/import-transcript.test.ts +++ b/packages/cli/importers/import-transcript.test.ts @@ -2,9 +2,9 @@ * Unit tests for importTranscriptFile — the live-capture (Claude hook) path. * * Uses a fake importer (parseFile returns a synthetic session) + an in-memory - * mock client that round-trips meta through the real buildMeta, so the - * watermark / incremental-delta / reconcile-fallback logic is exercised without - * a database. + * mock client that round-trips meta through the real buildMeta and simulates + * the server's conditional upsert, so the watermark / incremental-delta / + * version-bump re-render logic is exercised without a database. */ import { describe, expect, test } from "bun:test"; import type { MemoryClient } from "../client.ts"; @@ -28,11 +28,11 @@ const WRITE: WriteOptions = { verbose: false, }; -/** A mock engine backed by an in-memory id→meta store, mimicking the server. */ +/** A mock engine backed by an in-memory id→row store, mimicking the server. */ function mockEngine() { const store = new Map< string, - { id: string; meta: Record } + { id: string; meta: Record; content: string } >(); const client = { memory: { @@ -45,16 +45,33 @@ function mockEngine() { const limit = p.limit ?? 10; return { results: all.slice(0, limit), total: all.length, limit }; }, + // The server's conditional upsert: insert new ids; replace an existing + // row when its meta value for `replaceIfMetaDiffers` differs; else skip. batchCreate: async (p: { - memories: Array<{ id: string; meta: Record }>; + memories: Array<{ + id: string; + meta: Record; + content: string; + }>; + replaceIfMetaDiffers?: string; }) => { const ids: string[] = []; + const updatedIds: string[] = []; for (const m of p.memories) { - if (store.has(m.id)) throw new Error(`duplicate id ${m.id}`); - store.set(m.id, { id: m.id, meta: m.meta }); - ids.push(m.id); + const existing = store.get(m.id); + if (!existing) { + store.set(m.id, { id: m.id, meta: m.meta, content: m.content }); + ids.push(m.id); + } else if ( + p.replaceIfMetaDiffers !== undefined && + existing.meta[p.replaceIfMetaDiffers] !== + m.meta[p.replaceIfMetaDiffers] + ) { + store.set(m.id, { id: m.id, meta: m.meta, content: m.content }); + updatedIds.push(m.id); + } } - return { ids }; + return { ids, updatedIds }; }, }, } as unknown as MemoryClient; @@ -202,4 +219,33 @@ describe("importTranscriptFile", () => { expect(out?.inserted).toBe(0); expect(store.size).toBe(3); }); + + test("a stale importer_version is re-rendered in place (server-side upsert)", async () => { + const { client, store } = mockEngine(); + const s = session(["a", "b", "c"]); + await importTranscriptFile(client, importerFor(s), "/x.jsonl", WRITE); + expect(store.size).toBe(3); + + // Simulate rows written by an older importer build. + for (const row of store.values()) { + row.meta = { ...row.meta, importer_version: "0" }; + row.content = "stale render"; + } + + // The high-water row is stale → no narrowing; the full plan is submitted + // and the server's upsert rewrites every stale row in one pass. + const out = await importTranscriptFile( + client, + importerFor(s), + "/x.jsonl", + WRITE, + ); + expect(out?.updated).toBe(3); + expect(out?.inserted).toBe(0); + expect(out?.skipped).toBe(0); + for (const row of store.values()) { + expect(row.meta.importer_version).toBe("1"); + expect(row.content).not.toBe("stale render"); + } + }); }); diff --git a/packages/cli/importers/index.ts b/packages/cli/importers/index.ts index f007a6a..cfc89db 100644 --- a/packages/cli/importers/index.ts +++ b/packages/cli/importers/index.ts @@ -7,11 +7,15 @@ * writes one memory per message, using deterministic UUIDv7s keyed * by `(tool, sessionId, messageId)` so re-imports are idempotent. * - * Performance shape: each session does at most two RPCs against the - * engine — one `memory.search` to fetch existing message ids for the - * session, and one `memory.batchCreate` for everything new. Updates - * (only triggered by an `importer_version` bump) are issued one at a - * time and are expected to be rare. + * Reconciliation happens server-side: every planned message is submitted + * through the conditional upsert (`memory.batchCreate` with + * `replaceIfMetaDiffers: "importer_version"`) — new ids insert, rows whose + * stored `importer_version` differs are rewritten in place, and + * already-current rows are skipped, all classified from the batch + * response. No existing-state pre-fetch, so sessions of any size (including + * past the 1000-row search page) reconcile exactly. Per session that is + * ceil(n/chunk) `memory.batchCreate` calls; the live-capture hook adds one + * `memory.search` to narrow the submission to the new suffix. */ import type { MemoryCreateParams } from "@memory.build/protocol/memory"; @@ -30,21 +34,18 @@ import { deterministicMessageUuidV7 } from "./uuid.ts"; /** * Version tag stored in `meta.importer_version`. Bumping this forces a - * re-render of every previously-imported message on the next run (via - * the version check in `planSession`) so parser changes propagate - * without manual intervention. + * re-render of every previously-imported message on the next run: the + * server's conditional upsert replaces any row whose stored value for + * `IMPORTER_VERSION_KEY` differs from the submitted one, so parser changes + * propagate without manual intervention. * * Locked at "1" during pre-release iteration — bump only after the first * real release so early adopters get parser fixes without a manual wipe. */ export const IMPORTER_VERSION = "1"; -/** - * Maximum memories per `memory.search` lookup. Same protocol limit. A - * session with more existing messages than this triggers a fallback to - * paged lookups (rare in practice). - */ -const SEARCH_PAGE_LIMIT = 1000; +/** The meta key the server compares for the conditional replace. */ +const IMPORTER_VERSION_KEY = "importer_version"; /** * Default capture layout, shared by `me import claude` and the Claude Code capture @@ -186,10 +187,11 @@ export async function runImport( * * Incremental + stateless: it asks the server for the session's high-water * message (`searchSessionHighWater` — one `limit 1`, newest-first search) and - * writes only the messages after it. Falls back to the full reconcile - * (`writeSession`) for a new session, an `importer_version` bump, a lost anchor - * (compaction/reorder), or any fast-path write error — so correctness never - * depends on the optimization. Returns null when the file has no session. + * submits only the messages after it. The narrowing is purely a bandwidth + * optimization — a new session, an `importer_version` bump, or a lost anchor + * (compaction/reorder) submits the full plan, and the server's conditional + * upsert reconciles whatever overlaps. Returns null when the file has no + * session. */ export async function importTranscriptFile( engine: MemoryClient, @@ -209,8 +211,21 @@ export async function importTranscriptFile( session.cwd, ); const tree = `${options.treeRoot}.${slug}.${options.sessionsNodeName}`; - const title = synthesizeTitle(session); + const plan = planSession(session, tree, slug, gitRoot, gitRemote, options); + const outcome: SessionOutcome = { + sessionId: session.sessionId, + title: synthesizeTitle(session), + tree, + sourceFile: session.sourceFile, + inserted: 0, + updated: 0, + skipped: plan.skipped, + failed: plan.failed, + errors: [...plan.errors], + }; + + let planned = plan.planned; const hw = await searchSessionHighWater( engine, tree, @@ -218,56 +233,17 @@ export async function importTranscriptFile( session.sessionId, ); if (hw && hw.importerVersion === IMPORTER_VERSION) { - const plan = planSession(session, tree, slug, gitRoot, gitRemote, options); - const idx = plan.planned.findIndex( - (p) => p.message.messageId === hw.messageId, - ); + const idx = planned.findIndex((p) => p.message.messageId === hw.messageId); if (idx !== -1) { - const delta = plan.planned.slice(idx + 1); - const outcome: SessionOutcome = { - sessionId: session.sessionId, - title, - tree, - sourceFile: session.sourceFile, - inserted: 0, - updated: 0, - skipped: plan.skipped, - failed: plan.failed, - errors: [...plan.errors], - }; - if (delta.length === 0) return outcome; - if (options.dryRun) { - outcome.inserted += delta.length; - return outcome; - } - try { - const { insertedIds, errors } = await batchCreateChunked( - engine, - delta.map((d) => d.payload), - ); - if (errors.length === 0) { - // An already-present id (non-monotonic transcript) is silently - // skipped server-side, so inserted may be < delta.length. - outcome.inserted += insertedIds.length; - return outcome; - } - // A failed chunk → fall through to the full reconcile for correctness. - } catch { - // Unexpected write error → reconcile. - } + // The anchor and everything before it are already imported at the + // current version (transcripts are append-only). + outcome.skipped += idx + 1; + planned = planned.slice(idx + 1); } } - return writeSession( - engine, - session, - title, - tree, - slug, - gitRoot, - gitRemote, - options, - ); + await submitPlanned(engine, planned, outcome, options); + return outcome; } /** @@ -295,19 +271,6 @@ async function searchSessionHighWater( return { messageId, importerVersion: typeof v === "string" ? v : undefined }; } -/** - * Write all messages for one session. - * - * Strategy: - * 1. One `memory.search` to fetch existing message ids + their - * `importer_version` for this session. - * 2. Diff each rendered message against the existing set: - * - id absent → queue for batch insert - * - id present, ver matches → skip - * - id present, ver differs → queue for update - * 3. Issue one `memory.batchCreate` (in chunks of 1000) for inserts; - * updates are issued one at a time (rare path). - */ /** One planned message write (post-render, pre-dedup/diff). */ interface PlannedMessage { message: ConversationMessage; @@ -390,6 +353,11 @@ function planSession( }; } +/** + * Write all messages for one session: plan + dedup, then submit everything + * through the server's conditional upsert (see `submitPlanned`). No + * existing-state read — classification comes from the batch response. + */ async function writeSession( engine: MemoryClient, session: ImportedSession, @@ -423,126 +391,57 @@ async function writeSession( outcome.skipped += plan.skipped; outcome.failed += plan.failed; outcome.errors.push(...plan.errors); - const deduped = plan.planned; - - if (deduped.length === 0) return outcome; - - // Bulk-fetch existing message ids for this session in one search. - let existing: Map; - try { - existing = await fetchExistingMessageVersions(engine, session, tree); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - // The whole session fails if we can't determine existing state. - outcome.failed += deduped.length; - for (const p of deduped) { - outcome.errors.push({ - messageId: p.message.messageId, - error: `existing-state lookup failed: ${msg}`, - }); - } - return outcome; - } - - const toInsert: MemoryCreateParams[] = []; - const toUpdate: Array<{ messageId: string; payload: MemoryCreateParams }> = - []; - - for (const p of deduped) { - const existingVersion = existing.get(p.memoryId); - if (existingVersion === undefined) { - toInsert.push(p.payload); - } else if (existingVersion === IMPORTER_VERSION) { - outcome.skipped++; - } else { - toUpdate.push({ messageId: p.message.messageId, payload: p.payload }); - } - } - - // Inserts: one batchCreate per chunk. Chunks are cut by byte budget OR - // count cap, whichever fires first, so a chunk's serialized request body - // stays under the server's request size limit. - if (toInsert.length > 0) { - if (options.dryRun) { - outcome.inserted += toInsert.length; - } else { - const { insertedIds, errors } = await batchCreateChunked( - engine, - toInsert, - ); - outcome.inserted += insertedIds.length; - // Each chunk error contributes its full itemCount to `failed` and - // attaches the same message to each id in that chunk — matching the - // pre-chunking behavior of one error row per attempted message. - for (const e of errors) { - outcome.failed += e.itemCount; - for (const id of e.ids) { - outcome.errors.push({ messageId: id, error: e.error }); - } - } - } - } - - // Updates: rare, issued one at a time. - for (const u of toUpdate) { - if (options.dryRun) { - outcome.updated++; - continue; - } - try { - await engine.memory.update({ - id: u.payload.id as string, - content: u.payload.content, - meta: u.payload.meta, - tree: u.payload.tree, - temporal: u.payload.temporal, - }); - outcome.updated++; - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - outcome.failed++; - outcome.errors.push({ messageId: u.messageId, error: msg }); - } - } + await submitPlanned(engine, plan.planned, outcome, options); return outcome; } /** - * Fetch existing message ids + their `importer_version` for one session. + * Submit planned messages through the conditional upsert and fold the + * outcome into `outcome`: new ids insert, rows whose stored + * `importer_version` differs are rewritten in place (the version-bump + * re-render), and already-current rows are skipped — all classified from + * the batch response, independent of how many messages the session already + * has server-side. * - * Uses `memory.search` with a `meta` filter on `source_session_id` + - * `source_tool`. Restricted to the session's tree so the search is - * indexed by tree first. Returns `id → importer_version` (version is - * `undefined` when the record was written before the field existed). + * Chunks are cut by byte budget OR count cap (see batchCreateChunked) so + * each request body stays under the server's size limit; a failed chunk + * contributes its full itemCount to `failed` with one error row per id. + * + * Dry runs report every planned message as an insert — there is no server + * classification without submitting. */ -async function fetchExistingMessageVersions( +async function submitPlanned( engine: MemoryClient, - session: ImportedSession, - tree: string, -): Promise> { - const result = await engine.memory.search({ - tree, - meta: { - source_tool: session.tool, - source_session_id: session.sessionId, - }, - limit: SEARCH_PAGE_LIMIT, - }); - if (result.total > result.results.length) { - // Sessions exceeding 1000 already-imported messages would silently - // re-insert and hit duplicate-id errors. Surface that loudly so we - // can paginate when it actually happens. - throw new Error( - `session has ${result.total} existing messages but bulk-fetch is capped at ${SEARCH_PAGE_LIMIT}; pagination not yet implemented`, - ); + planned: PlannedMessage[], + outcome: SessionOutcome, + options: WriteOptions, +): Promise { + if (planned.length === 0) return; + if (options.dryRun) { + outcome.inserted += planned.length; + return; } - const map = new Map(); - for (const r of result.results) { - const v = r.meta.importer_version; - map.set(r.id, typeof v === "string" ? v : undefined); + + const { insertedIds, updatedIds, errors } = await batchCreateChunked( + engine, + planned.map((p) => p.payload), + { replaceIfMetaDiffers: IMPORTER_VERSION_KEY }, + ); + outcome.inserted += insertedIds.length; + outcome.updated += updatedIds.length; + let failedCount = 0; + for (const e of errors) { + failedCount += e.itemCount; + outcome.failed += e.itemCount; + for (const id of e.ids) { + outcome.errors.push({ messageId: id, error: e.error }); + } } - return map; + // Whatever the server neither inserted, updated, nor failed already + // exists at the current importer_version. + outcome.skipped += + planned.length - insertedIds.length - updatedIds.length - failedCount; } /** Build the full meta object for one message memory. */ @@ -565,7 +464,7 @@ function buildMeta( source_file: session.sourceFile, content_mode: options.fullTranscript ? "full_transcript" : "default", imported_at: new Date().toISOString(), - importer_version: IMPORTER_VERSION, + [IMPORTER_VERSION_KEY]: IMPORTER_VERSION, }; if (session.title) meta.source_session_title = session.title; diff --git a/packages/cli/util.ts b/packages/cli/util.ts index e9116ee..e658965 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -104,6 +104,10 @@ export function buildMemoryClient( url: creds.server, token: creds.apiKey ?? creds.sessionToken, space: creds.activeSpace, + // Bulk imports send 1000-memory batchCreate chunks that the server + // processes row-by-row; on a loaded server (or one far from its + // database) a chunk can legitimately exceed the client's 30s default. + timeout: 120_000, }); } diff --git a/packages/database/space/migrate/idempotent/001_memory.sql b/packages/database/space/migrate/idempotent/001_memory.sql index 35b133f..01d4a33 100644 --- a/packages/database/space/migrate/idempotent/001_memory.sql +++ b/packages/database/space/migrate/idempotent/001_memory.sql @@ -100,10 +100,27 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ------------------------------------------------------------------------------- -- create memory -- --- Returns the new memory's id, or null when an explicit _id already exists --- (on conflict do nothing). The null lets importers with deterministic ids --- re-submit safely — the caller classifies a missing id as "skipped". +-- Insert one memory. On a duplicate explicit _id the outcome depends on +-- _replace_if_meta_differs: +-- - null (default): skip — the existing row is left untouched. +-- - a meta key name: the existing row is REPLACED (tree/meta/temporal/ +-- content) when its meta->>key value differs from the new record's, and +-- skipped when it matches. Deterministic-id importers use this to push +-- re-renders by bumping a version value in meta (importer_version). +-- The replace arm additionally requires write access on the EXISTING +-- row's tree; without it the row is silently skipped (not raised, unlike +-- patch_memory) so one inaccessible row can't fail a whole batch. +-- +-- Returns zero rows on a skip, else one row (id, inserted) where inserted +-- distinguishes a fresh insert (true) from a replace (false; detected via +-- xmax = 0). Embedding columns are never set here: the update triggers +-- invalidate and re-enqueue the embedding only when content actually changed, +-- so a meta-only replace does not re-embed. +-- +-- The drop covers the pre-upsert 6-arg signature — without it, create would +-- add an ambiguous overload (and the return type changed). No-op on re-runs. ------------------------------------------------------------------------------- +drop function if exists {{schema}}.create_memory(jsonb, ltree, text, uuid, jsonb, tstzrange); create or replace function {{schema}}.create_memory ( _tree_access jsonb , _tree ltree @@ -111,16 +128,21 @@ create or replace function {{schema}}.create_memory , _id uuid default null , _meta jsonb default '{}' , _temporal tstzrange default null +, _replace_if_meta_differs text default null ) -returns uuid +returns table (id uuid, inserted boolean) as $func$ +-- The out columns (id, inserted) shadow table columns inside the body; the +-- body never reads them as variables, so resolve ambiguity to the columns. +#variable_conflict use_column begin if not {{schema}}.has_tree_access(_tree_access, _tree, 2) then raise exception 'insufficient tree access' using errcode = 'insufficient_privilege'; end if; - insert into {{schema}}.memory + return query + insert into {{schema}}.memory as m ( id , tree , meta @@ -134,10 +156,17 @@ begin , _temporal , _content ) - on conflict (id) do nothing - returning id into _id + on conflict (id) do update set + tree = excluded.tree + , meta = excluded.meta + , temporal = excluded.temporal + , content = excluded.content + where _replace_if_meta_differs is not null + and m.meta->>_replace_if_meta_differs + is distinct from excluded.meta->>_replace_if_meta_differs + and {{schema}}.has_tree_access(_tree_access, m.tree, 2) + returning m.id, (m.xmax = 0) ; - return _id; end; $func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp diff --git a/packages/database/space/migrate/migrate.integration.test.ts b/packages/database/space/migrate/migrate.integration.test.ts index cc58bf8..f294863 100644 --- a/packages/database/space/migrate/migrate.integration.test.ts +++ b/packages/database/space/migrate/migrate.integration.test.ts @@ -286,22 +286,26 @@ describe("provisioned schema is functional", () => { expect(updated?.updated_at).not.toBeNull(); }); - test("create_memory skips a duplicate explicit id (returns null)", async () => { - // Deterministic-id importers re-submit existing ids; the second create - // must be a no-op that returns null, leaving the original row intact. - const owner = `'[{"tree_path": "", "access": 3}]'::jsonb`; + // create_memory's conditional upsert: (treeAccess, tree, content, id, meta, + // temporal, replaceIfMetaDiffers) → zero rows (skip) or (id, inserted). + const OWNER = `'[{"tree_path": "", "access": 3}]'::jsonb`; + const createMemory = (args: string) => + sql.unsafe(`select * from ${canonical.schema}.create_memory(${args})`); + + test("create_memory skips a duplicate explicit id by default", async () => { + // Deterministic-id importers re-submit existing ids; with no replace key + // the second create must be a zero-row no-op leaving the row intact. const id = "01941000-0000-7000-8000-000000000001"; - const [first] = await sql.unsafe( - `select ${canonical.schema}.create_memory( - ${owner}, 'a.dup'::ltree, 'original', '${id}'::uuid) as id`, + const [first] = await createMemory( + `${OWNER}, 'a.dup'::ltree, 'original', '${id}'::uuid`, ); expect(first?.id).toBe(id); + expect(first?.inserted).toBe(true); - const [second] = await sql.unsafe( - `select ${canonical.schema}.create_memory( - ${owner}, 'a.dup'::ltree, 'replacement', '${id}'::uuid) as id`, + const second = await createMemory( + `${OWNER}, 'a.dup'::ltree, 'replacement', '${id}'::uuid`, ); - expect(second?.id).toBeNull(); + expect(second.length).toBe(0); const [row] = await sql.unsafe( `select content from ${canonical.schema}.memory where id = '${id}'`, @@ -309,6 +313,108 @@ describe("provisioned schema is functional", () => { expect(row?.content).toBe("original"); }); + test("create_memory replaces a duplicate when the meta key differs, skips when it matches", async () => { + const id = "01941000-0000-7000-8000-000000000002"; + await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v1', '${id}'::uuid, '{"v": "1"}'::jsonb`, + ); + + // Same version → skip, content untouched. + const same = await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v1 again', '${id}'::uuid, '{"v": "1"}'::jsonb, null, 'v'`, + ); + expect(same.length).toBe(0); + + // Bumped version → replaced in place, inserted = false. + const [bumped] = await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v2', '${id}'::uuid, '{"v": "2"}'::jsonb, null, 'v'`, + ); + expect(bumped?.id).toBe(id); + expect(bumped?.inserted).toBe(false); + + const [row] = await sql.unsafe( + `select content, meta->>'v' as v, updated_at + from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("render v2"); + expect(row?.v).toBe("2"); + expect(row?.updated_at).not.toBeNull(); + + // A key absent on the stored row but present on the new record counts as + // "differs" (legacy rows written before the version key existed). + const [legacy] = await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v3', '${id}'::uuid, '{"v": "2", "legacy_v": "1"}'::jsonb, null, 'legacy_v'`, + ); + expect(legacy?.inserted).toBe(false); + const [afterLegacy] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(afterLegacy?.content).toBe("render v3"); + }); + + test("create_memory replace requires write access on the existing row's tree", async () => { + // Row lives under a.secret; the caller's grant covers only a.open — the + // insert-arm check passes (target a.open) but the replace arm must skip. + const id = "01941000-0000-7000-8000-000000000003"; + await createMemory( + `${OWNER}, 'a.secret'::ltree, 'guarded', '${id}'::uuid, '{"v": "1"}'::jsonb`, + ); + + const limited = `'[{"tree_path": "a.open", "access": 3}]'::jsonb`; + const res = await createMemory( + `${limited}, 'a.open'::ltree, 'hijack', '${id}'::uuid, '{"v": "2"}'::jsonb, null, 'v'`, + ); + expect(res.length).toBe(0); + + const [row] = await sql.unsafe( + `select content, tree::text as tree from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("guarded"); + expect(row?.tree).toBe("a.secret"); + }); + + test("create_memory replace re-embeds only when content changed", async () => { + const id = "01941000-0000-7000-8000-000000000004"; + await createMemory( + `${OWNER}, 'a.emb'::ltree, 'stable content', '${id}'::uuid, '{"v": "1"}'::jsonb`, + ); + // Simulate the worker: embedding present (default 1536 dims), queue drained. + await sql.unsafe( + `update ${canonical.schema}.memory + set embedding = ('[' || repeat('0,', 1535) || '0]')::halfvec + where id = '${id}'`, + ); + await sql.unsafe( + `delete from ${canonical.schema}.embedding_queue where memory_id = '${id}'`, + ); + + // Meta-only replace (identical content): embedding survives, no re-enqueue. + await createMemory( + `${OWNER}, 'a.emb'::ltree, 'stable content', '${id}'::uuid, '{"v": "2"}'::jsonb, null, 'v'`, + ); + const [afterMeta] = await sql.unsafe( + `select (embedding is not null) as has_embedding, + (select count(*)::int from ${canonical.schema}.embedding_queue + where memory_id = '${id}' and outcome is null) as queued + from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(afterMeta?.has_embedding).toBe(true); + expect(afterMeta?.queued).toBe(0); + + // Content replace: embedding invalidated and re-enqueued. + await createMemory( + `${OWNER}, 'a.emb'::ltree, 'new content', '${id}'::uuid, '{"v": "3"}'::jsonb, null, 'v'`, + ); + const [afterContent] = await sql.unsafe( + `select (embedding is null) as embedding_cleared, + (select count(*)::int from ${canonical.schema}.embedding_queue + where memory_id = '${id}' and outcome is null) as queued + from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(afterContent?.embedding_cleared).toBe(true); + expect(afterContent?.queued).toBe(1); + }); + test("enforces the meta-is-object constraint", async () => { await expectReject(() => sql.unsafe( diff --git a/packages/engine/space/db.integration.test.ts b/packages/engine/space/db.integration.test.ts index faae91c..9063ae6 100644 --- a/packages/engine/space/db.integration.test.ts +++ b/packages/engine/space/db.integration.test.ts @@ -51,14 +51,15 @@ async function setEmbedding(id: string, vec: number[]): Promise { ); } -/** createMemory asserting the insert happened (no duplicate-id skip). */ +/** createMemory asserting a fresh insert happened (no skip/replace). */ async function mustCreate( access: TreeAccess, params: Parameters[1], ): Promise { - const id = await db.createMemory(access, params); - if (id === null) throw new Error("unexpected duplicate-id skip"); - return id; + const created = await db.createMemory(access, params); + if (created === null) throw new Error("unexpected duplicate-id skip"); + if (!created.inserted) throw new Error("unexpected replace"); + return created.id; } test("createMemory + getMemory round-trips", async () => { @@ -82,7 +83,7 @@ test("createMemory returns null for a duplicate explicit id", async () => { tree: "work.dup", content: "original", }); - expect(first).toBe(id); + expect(first).toEqual({ id, inserted: true }); // Re-submitting the same id is a no-op skip, not an error. const second = await db.createMemory(FULL, { @@ -94,6 +95,40 @@ test("createMemory returns null for a duplicate explicit id", async () => { expect((await db.getMemory(FULL, id))?.content).toBe("original"); }); +test("createMemory with replaceIfMetaDiffers rewrites stale rows in place", async () => { + const id = "01900000-0000-7000-8000-0000000000d1"; + await db.createMemory(FULL, { + id, + tree: "work.upsert", + content: "render v1", + meta: { importer_version: "1" }, + }); + + // Same version → skip. + const same = await db.createMemory(FULL, { + id, + tree: "work.upsert", + content: "render v1 again", + meta: { importer_version: "1" }, + replaceIfMetaDiffers: "importer_version", + }); + expect(same).toBeNull(); + expect((await db.getMemory(FULL, id))?.content).toBe("render v1"); + + // Bumped version → replaced, reported as an update (inserted: false). + const bumped = await db.createMemory(FULL, { + id, + tree: "work.upsert", + content: "render v2", + meta: { importer_version: "2" }, + replaceIfMetaDiffers: "importer_version", + }); + expect(bumped).toEqual({ id, inserted: false }); + const after = await db.getMemory(FULL, id); + expect(after?.content).toBe("render v2"); + expect(after?.meta).toEqual({ importer_version: "2" }); +}); + test("access is enforced by the tree_access argument", async () => { // create requires write (>=2): read-only access is rejected await expect( diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts index 2dbd157..957eb87 100644 --- a/packages/engine/space/db.ts +++ b/packages/engine/space/db.ts @@ -20,14 +20,17 @@ import type { */ export interface SpaceStore { /** - * Insert one memory, returning its id — or null when an explicit - * `params.id` already exists (`on conflict do nothing`), so deterministic-id - * importers can re-submit idempotently. + * Insert one memory. When an explicit `params.id` already exists the + * outcome depends on `params.replaceIfMetaDiffers`: unset → skip (null); + * set to a meta key → the existing row is replaced when its value for that + * key differs from the new record's (`inserted: false`), else skipped. + * Deterministic-id importers use this to re-submit idempotently and push + * version-bump re-renders in the same call. */ createMemory( treeAccess: TreeAccess, params: CreateMemoryParams, - ): Promise; + ): Promise<{ id: string; inserted: boolean } | null>; getMemory(treeAccess: TreeAccess, id: string): Promise; patchMemory( treeAccess: TreeAccess, @@ -118,17 +121,19 @@ export function spaceStore(sql: Sql, schema: string): SpaceStore { return { async createMemory(treeAccess, p) { const [row] = await sql` - select ${sch}.create_memory( + select id, inserted from ${sch}.create_memory( ${jb(treeAccess)}, ${p.tree}::ltree, ${p.content}, ${p.id ?? null}, ${jb(p.meta)}, - ${p.temporal ?? null}::tstzrange - ) as id`; - if (!row) throw new Error("create_memory returned no row"); - // Null id = the explicit id already exists (on conflict do nothing). - return (row.id as string | null) ?? null; + ${p.temporal ?? null}::tstzrange, + ${p.replaceIfMetaDiffers ?? null} + )`; + // Zero rows = the explicit id already exists and was skipped (version + // match, no replace key, or no write access on the existing row's tree). + if (!row) return null; + return { id: row.id as string, inserted: Boolean(row.inserted) }; }, async getMemory(treeAccess, id) { diff --git a/packages/engine/space/types.ts b/packages/engine/space/types.ts index b11f10e..215c17f 100644 --- a/packages/engine/space/types.ts +++ b/packages/engine/space/types.ts @@ -34,6 +34,12 @@ export interface CreateMemoryParams { id?: string; meta?: Record; temporal?: TemporalRange; + /** + * Meta key for conditional replace: when an explicit `id` already exists, + * replace the row iff its meta value for this key differs from the new + * record (e.g. importer_version). Default: duplicates are skipped. + */ + replaceIfMetaDiffers?: string; } export interface MemoryPatch { diff --git a/packages/protocol/memory.ts b/packages/protocol/memory.ts index 068f982..42286ac 100644 --- a/packages/protocol/memory.ts +++ b/packages/protocol/memory.ts @@ -31,6 +31,12 @@ export type MemoryCreateParams = z.infer; /** * memory.batchCreate params. + * + * `replaceIfMetaDiffers` names a meta key for conditional replace: a memory + * whose explicit id already exists is rewritten in place when the stored + * row's value for that key differs from the submitted one (deterministic-id + * importers pass e.g. "importer_version" so version bumps re-render existing + * rows), and skipped when it matches. Without it, duplicates are skipped. */ export const memoryBatchCreateParams = z.object({ memories: z @@ -45,6 +51,7 @@ export const memoryBatchCreateParams = z.object({ ) .min(1, "at least one memory required") .max(1000, "maximum 1000 memories per batch"), + replaceIfMetaDiffers: z.string().min(1).optional().nullable(), }); export type MemoryBatchCreateParams = z.infer; @@ -176,9 +183,15 @@ export type MemoryWithScoreResponse = z.infer; /** * memory.batchCreate result. + * + * `ids` are the freshly inserted memories; `updatedIds` are existing rows + * rewritten in place via `replaceIfMetaDiffers`. A submitted explicit id in + * neither array (and not in a failed request) was skipped — it already + * existed at the same meta-key value. */ export const memoryBatchCreateResult = z.object({ ids: z.array(z.string()), + updatedIds: z.array(z.string()), }); export type MemoryBatchCreateResult = z.infer; diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts index cb714f7..d3bdce0 100644 --- a/packages/server/rpc/memory/memory.integration.test.ts +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -248,20 +248,84 @@ test("get / delete unknown id → NOT_FOUND", async () => { }); test("batchCreate inserts all and is retrievable", async () => { - const res = await call<{ ids: string[] }>("memory.batchCreate", { - memories: [ - { content: "one", tree: "share.batch" }, - { content: "two", tree: "share.batch" }, - { content: "three", tree: "share.batch.sub" }, - ], - }); + const res = await call<{ ids: string[]; updatedIds: string[] }>( + "memory.batchCreate", + { + memories: [ + { content: "one", tree: "share.batch" }, + { content: "two", tree: "share.batch" }, + { content: "three", tree: "share.batch.sub" }, + ], + }, + ); expect(res.ids).toHaveLength(3); + expect(res.updatedIds).toHaveLength(0); const count = await call<{ count: number }>("memory.countTree", { tree: "share.batch", }); expect(count.count).toBe(3); }); +test("create with a duplicate explicit id → CONFLICT", async () => { + const id = "01941000-0000-7000-8000-00000000c0f1"; + await call("memory.create", { id, content: "first", tree: "share.dup" }); + await expectAppError( + call("memory.create", { id, content: "second", tree: "share.dup" }), + "CONFLICT", + ); +}); + +test("batchCreate without replaceIfMetaDiffers skips duplicates", async () => { + const id = "01941000-0000-7000-8000-00000000c0f2"; + await call("memory.batchCreate", { + memories: [{ id, content: "original", tree: "share.skip" }], + }); + const res = await call<{ ids: string[]; updatedIds: string[] }>( + "memory.batchCreate", + { memories: [{ id, content: "replacement", tree: "share.skip" }] }, + ); + expect(res.ids).toHaveLength(0); + expect(res.updatedIds).toHaveLength(0); + const got = await call<{ content: string }>("memory.get", { id }); + expect(got.content).toBe("original"); +}); + +test("batchCreate with replaceIfMetaDiffers splits insert/update/skip", async () => { + const stale = "01941000-0000-7000-8000-00000000c0f3"; + const fresh = "01941000-0000-7000-8000-00000000c0f4"; + const brandNew = "01941000-0000-7000-8000-00000000c0f5"; + await call("memory.batchCreate", { + memories: [ + { id: stale, content: "old render", tree: "share.up", meta: { v: "1" } }, + { id: fresh, content: "current", tree: "share.up", meta: { v: "2" } }, + ], + }); + + const res = await call<{ ids: string[]; updatedIds: string[] }>( + "memory.batchCreate", + { + memories: [ + { + id: stale, + content: "new render", + tree: "share.up", + meta: { v: "2" }, + }, + { id: fresh, content: "untouched", tree: "share.up", meta: { v: "2" } }, + { id: brandNew, content: "added", tree: "share.up", meta: { v: "2" } }, + ], + replaceIfMetaDiffers: "v", + }, + ); + expect(res.ids).toEqual([brandNew]); + expect(res.updatedIds).toEqual([stale]); + + const updated = await call<{ content: string }>("memory.get", { id: stale }); + expect(updated.content).toBe("new render"); + const skipped = await call<{ content: string }>("memory.get", { id: fresh }); + expect(skipped.content).toBe("current"); +}); + test("tree returns descendant node counts under a path", async () => { await call("memory.batchCreate", { memories: [ diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 6256373..251148b 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -177,7 +177,7 @@ async function memoryCreate( const ctx = context as SpaceRpcContext; const { store, treeAccess } = ctx; - const id = await guard(() => + const created = await guard(() => store.createMemory(treeAccess, { id: params.id ?? undefined, content: params.content, @@ -186,12 +186,12 @@ async function memoryCreate( temporal: formatTemporal(params.temporal), }), ); - if (id === null) { - // The store skips an explicit id that already exists (on conflict do - // nothing). For a single create that's a caller error, not a skip. + if (created === null) { + // The store skips an explicit id that already exists (no replace key is + // passed here). For a single create that's a caller error, not a skip. throw new AppError("CONFLICT", `Memory already exists: ${params.id}`); } - const memory = await store.getMemory(treeAccess, id); + const memory = await store.getMemory(treeAccess, created.id); if (!memory) { throw new AppError("INTERNAL_ERROR", "Created memory could not be read"); } @@ -201,10 +201,11 @@ async function memoryCreate( /** * memory.batchCreate — atomic across the batch. * - * Returns the inserted ids only: a memory whose explicit id already exists - * is silently skipped (`on conflict do nothing` in create_memory), so - * deterministic-id importers can re-submit and classify the missing ids as - * already imported (see `computeSkippedIds` in the CLI). + * `ids` carries the inserted memories; `updatedIds` the existing rows + * rewritten via `replaceIfMetaDiffers` (conditional upsert in + * create_memory). A submitted explicit id in neither array was skipped — + * deterministic-id importers re-submit freely and classify the missing ids + * as already imported. */ async function memoryBatchCreate( params: MemoryBatchCreateParams, @@ -214,23 +215,25 @@ async function memoryBatchCreate( const ctx = context as SpaceRpcContext; const { store, treeAccess } = ctx; - const ids = await guard(() => + return await guard(() => store.withTransaction(async (tx) => { - const out: string[] = []; + const ids: string[] = []; + const updatedIds: string[] = []; for (const m of params.memories) { - const id = await tx.createMemory(treeAccess, { + const created = await tx.createMemory(treeAccess, { id: m.id ?? undefined, content: m.content, meta: m.meta ?? undefined, tree: inputTreePath(ctx, m.tree), temporal: formatTemporal(m.temporal), + replaceIfMetaDiffers: params.replaceIfMetaDiffers ?? undefined, }); - if (id !== null) out.push(id); + if (created === null) continue; + (created.inserted ? ids : updatedIds).push(created.id); } - return out; + return { ids, updatedIds }; }), ); - return { ids }; } /** memory.get */ diff --git a/packages/server/start.integration.test.ts b/packages/server/start.integration.test.ts index bed8002..2d95ca0 100644 --- a/packages/server/start.integration.test.ts +++ b/packages/server/start.integration.test.ts @@ -158,5 +158,5 @@ test("re-migrates existing space schemas on boot", async () => { expect(tamperedDef).not.toContain("on conflict"); // …and boot's space sweep re-applied the idempotent SQL over it. expect(bootedDef).not.toContain("stale stand-in"); - expect(bootedDef).toContain("on conflict (id) do nothing"); + expect(bootedDef).toContain("on conflict (id) do update"); }); From bd2762e4655c9fda51fc58b8d684bf8b2332aaf5 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 19:36:32 +0200 Subject: [PATCH 137/156] =?UTF-8?q?perf(server):=20set-based=20batch=5Fcre?= =?UTF-8?q?ate=5Fmemory=20=E2=80=94=20one=20statement=20per=20chunk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit memory.batchCreate looped create_memory per row inside a transaction: a 1000-memory chunk was 1000 server↔database round trips, which is what made bulk imports slow (and was the reason the CLI client timeout had to be raised). New batch_create_memory takes the chunk as parallel arrays (uuid[]/ltree[]/text[]/tstzrange[] + one jsonb array for metas — a jsonb[] parameter double-encodes elements into string scalars, hence the single jsonb array aligned by ordinality via sql.json) and runs one unnest-driven INSERT ... ON CONFLICT DO UPDATE with identical per-row semantics. create_memory becomes a one-row wrapper over batch_create_memory, so the conflict rule (insert / replace-if-meta-differs / skip) lives in exactly one place; it is defined after the batch function since SQL-language bodies are validated at creation. Two deliberate semantic notes, both documented: the target-tree access check is all-or-nothing up front, and an explicit id repeated WITHIN a batch collapses to its first occurrence (a single INSERT cannot touch the same row twice; the importers already dedupe client-side). The handler drops its per-row loop and withTransaction (one statement is atomic by itself). Whale smoke (27 sessions, ~15.4K messages, local server against the remote ghost db): 345s → 55s per pass, zero failures, identical insert/skip classification. Co-Authored-By: Claude Fable 5 --- .../space/migrate/idempotent/001_memory.sql | 122 ++++++++++++++---- .../space/migrate/migrate.integration.test.ts | 94 ++++++++++++++ packages/engine/space/db.integration.test.ts | 29 +++++ packages/engine/space/db.ts | 32 +++++ packages/server/rpc/memory/memory.ts | 47 +++---- packages/server/start.integration.test.ts | 5 +- 6 files changed, 279 insertions(+), 50 deletions(-) diff --git a/packages/database/space/migrate/idempotent/001_memory.sql b/packages/database/space/migrate/idempotent/001_memory.sql index 01d4a33..7ebc54a 100644 --- a/packages/database/space/migrate/idempotent/001_memory.sql +++ b/packages/database/space/migrate/idempotent/001_memory.sql @@ -98,10 +98,12 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- create memory +-- batch create memory -- --- Insert one memory. On a duplicate explicit _id the outcome depends on --- _replace_if_meta_differs: +-- The canonical memory insert: one set-based statement for a whole batch +-- (create_memory below is a one-row wrapper). Parallel arrays, aligned by +-- position, carry the rows. Per-row, on a duplicate explicit id the outcome +-- depends on _replace_if_meta_differs: -- - null (default): skip — the existing row is left untouched. -- - a meta key name: the existing row is REPLACED (tree/meta/temporal/ -- content) when its meta->>key value differs from the new record's, and @@ -111,23 +113,24 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp -- row's tree; without it the row is silently skipped (not raised, unlike -- patch_memory) so one inaccessible row can't fail a whole batch. -- --- Returns zero rows on a skip, else one row (id, inserted) where inserted --- distinguishes a fresh insert (true) from a replace (false; detected via --- xmax = 0). Embedding columns are never set here: the update triggers --- invalidate and re-enqueue the embedding only when content actually changed, --- so a meta-only replace does not re-embed. +-- Returns one row (id, inserted) per insert/replace — inserted distinguishes +-- a fresh insert (true, xmax = 0) from a replace (false); skipped rows are +-- absent. The target-tree access check is all-or-nothing up front (one bad +-- row raises before anything is written), and an explicit id repeated WITHIN +-- the batch collapses to its first occurrence (a single INSERT cannot touch +-- the same row twice); later occurrences are skipped. -- --- The drop covers the pre-upsert 6-arg signature — without it, create would --- add an ambiguous overload (and the return type changed). No-op on re-runs. +-- Embedding columns are never set here: the update triggers invalidate and +-- re-enqueue the embedding only when content actually changed, so a +-- meta-only replace does not re-embed. ------------------------------------------------------------------------------- -drop function if exists {{schema}}.create_memory(jsonb, ltree, text, uuid, jsonb, tstzrange); -create or replace function {{schema}}.create_memory +create or replace function {{schema}}.batch_create_memory ( _tree_access jsonb -, _tree ltree -, _content text -, _id uuid default null -, _meta jsonb default '{}' -, _temporal tstzrange default null +, _ids uuid[] -- null elements get a generated uuidv7 +, _trees ltree[] +, _contents text[] +, _metas jsonb -- json ARRAY of meta objects; null elements default to '{}' +, _temporals tstzrange[] , _replace_if_meta_differs text default null ) returns table (id uuid, inserted boolean) @@ -136,12 +139,51 @@ as $func$ -- body never reads them as variables, so resolve ambiguity to the columns. #variable_conflict use_column begin - if not {{schema}}.has_tree_access(_tree_access, _tree, 2) then + -- _metas is one jsonb array (not jsonb[]): drivers pass json values + -- reliably (sql.json), where a jsonb[] parameter invites double-encoded + -- string scalars. Elements align with the arrays by position. + if jsonb_typeof(_metas) is distinct from 'array' + or cardinality(_ids) is distinct from cardinality(_trees) + or cardinality(_ids) is distinct from cardinality(_contents) + or cardinality(_ids) is distinct from jsonb_array_length(_metas) + or cardinality(_ids) is distinct from cardinality(_temporals) + then + raise exception 'batch arrays must have equal lengths' + using errcode = 'invalid_parameter_value'; + end if; + + if exists + ( + select 1 + from unnest(_trees) t(tree) + where not {{schema}}.has_tree_access(_tree_access, t.tree, 2) + ) then raise exception 'insufficient tree access' using errcode = 'insufficient_privilege'; end if; return query + with r as + ( + select + coalesce(u.id, uuidv7()) as id + , u.tree + , coalesce(nullif(e.meta, 'null'::jsonb), '{}'::jsonb) as meta + , u.temporal + , u.content + , u.ord + from unnest(_ids, _trees, _contents, _temporals) + with ordinality u(id, tree, content, temporal, ord) + join jsonb_array_elements(_metas) with ordinality e(meta, ord) + on e.ord = u.ord + ) + , d as + ( + -- First occurrence wins when a batch repeats an explicit id. + select distinct on (r.id) r.* + from r + order by r.id, r.ord + ) insert into {{schema}}.memory as m ( id , tree @@ -149,13 +191,8 @@ begin , temporal , content ) - values - ( coalesce(_id, uuidv7()) - , _tree - , coalesce(_meta, '{}'::jsonb) - , _temporal - , _content - ) + select d.id, d.tree, d.meta, d.temporal, d.content + from d on conflict (id) do update set tree = excluded.tree , meta = excluded.meta @@ -172,6 +209,41 @@ $func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; +------------------------------------------------------------------------------- +-- create memory +-- +-- One-row wrapper over batch_create_memory — see there for the conflict +-- semantics (insert / replace-if-meta-differs / skip) and the return shape. +-- +-- The drop covers the pre-upsert 6-arg signature — without it, create would +-- add an ambiguous overload (and the return type changed). No-op on re-runs. +------------------------------------------------------------------------------- +drop function if exists {{schema}}.create_memory(jsonb, ltree, text, uuid, jsonb, tstzrange); +create or replace function {{schema}}.create_memory +( _tree_access jsonb +, _tree ltree +, _content text +, _id uuid default null +, _meta jsonb default '{}' +, _temporal tstzrange default null +, _replace_if_meta_differs text default null +) +returns table (id uuid, inserted boolean) +as $func$ + select b.id, b.inserted + from {{schema}}.batch_create_memory( + _tree_access, + array[_id]::uuid[], + array[_tree], + array[_content], + jsonb_build_array(coalesce(_meta, '{}'::jsonb)), + array[_temporal], + _replace_if_meta_differs + ) b; +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + ------------------------------------------------------------------------------- -- patch memory ------------------------------------------------------------------------------- diff --git a/packages/database/space/migrate/migrate.integration.test.ts b/packages/database/space/migrate/migrate.integration.test.ts index f294863..09df1a1 100644 --- a/packages/database/space/migrate/migrate.integration.test.ts +++ b/packages/database/space/migrate/migrate.integration.test.ts @@ -40,6 +40,7 @@ const EXPECTED_TABLES = ["embedding_queue", "memory", "migration", "version"]; const EXPECTED_MIGRATIONS = ["001_memory", "002_embedding_queue"]; const EXPECTED_MEMORY_FUNCTIONS = [ + "batch_create_memory", "copy_tree", "count_tree", "create_memory", @@ -373,6 +374,99 @@ describe("provisioned schema is functional", () => { expect(row?.tree).toBe("a.secret"); }); + test("batch_create_memory upserts a whole batch in one statement", async () => { + const stale = "01941000-0000-7000-8000-00000000b001"; + const fresh = "01941000-0000-7000-8000-00000000b002"; + await createMemory( + `${OWNER}, 'a.batch'::ltree, 'old render', '${stale}'::uuid, '{"v": "1"}'::jsonb`, + ); + await createMemory( + `${OWNER}, 'a.batch'::ltree, 'current', '${fresh}'::uuid, '{"v": "2"}'::jsonb`, + ); + + // One call carrying: a stale row (update), a current row (skip), a brand + // new row (insert), and a no-id row (insert with generated id). + const rows = await sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${OWNER}, + array['${stale}', '${fresh}', '01941000-0000-7000-8000-00000000b003', null]::uuid[], + array['a.batch', 'a.batch', 'a.batch', 'a.batch']::ltree[], + array['new render', 'untouched', 'added', 'generated']::text[], + '[{"v": "2"}, {"v": "2"}, {"v": "2"}, {"v": "2"}]'::jsonb, + array[null, null, null, null]::tstzrange[], + 'v' + )`, + ); + const byId = new Map(rows.map((r) => [r.id as string, r.inserted])); + expect(byId.get(stale)).toBe(false); // replaced + expect(byId.has(fresh)).toBe(false); // skipped → absent + expect(byId.get("01941000-0000-7000-8000-00000000b003")).toBe(true); + expect(rows).toHaveLength(3); // 2 inserts + 1 update + + const [updated] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${stale}'`, + ); + expect(updated?.content).toBe("new render"); + const [skipped] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${fresh}'`, + ); + expect(skipped?.content).toBe("current"); + }); + + test("batch_create_memory collapses an id repeated within the batch (first wins)", async () => { + const id = "01941000-0000-7000-8000-00000000b010"; + const rows = await sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${OWNER}, + array['${id}', '${id}']::uuid[], + array['a.batchdup', 'a.batchdup']::ltree[], + array['first', 'second']::text[], + '[{}, {}]'::jsonb, + array[null, null]::tstzrange[] + )`, + ); + expect(rows).toHaveLength(1); + const [row] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("first"); + }); + + test("batch_create_memory rejects misaligned arrays and bad target access", async () => { + await expectReject(() => + sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${OWNER}, + array[null]::uuid[], + array['a.x', 'a.y']::ltree[], + array['one']::text[], + '[{}]'::jsonb, + array[null]::tstzrange[] + )`, + ), + ); + + // One row outside the grant fails the whole batch before any write. + const limited = `'[{"tree_path": "a.open", "access": 3}]'::jsonb`; + await expectReject(() => + sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${limited}, + array[null, null]::uuid[], + array['a.open', 'a.secret']::ltree[], + array['ok', 'denied']::text[], + '[{}, {}]'::jsonb, + array[null, null]::tstzrange[] + )`, + ), + ); + const [count] = await sql.unsafe( + `select count(*)::int as n from ${canonical.schema}.memory + where content in ('ok', 'denied')`, + ); + expect(count?.n).toBe(0); + }); + test("create_memory replace re-embeds only when content changed", async () => { const id = "01941000-0000-7000-8000-000000000004"; await createMemory( diff --git a/packages/engine/space/db.integration.test.ts b/packages/engine/space/db.integration.test.ts index 9063ae6..56e8532 100644 --- a/packages/engine/space/db.integration.test.ts +++ b/packages/engine/space/db.integration.test.ts @@ -129,6 +129,35 @@ test("createMemory with replaceIfMetaDiffers rewrites stale rows in place", asyn expect(after?.meta).toEqual({ importer_version: "2" }); }); +test("batchCreateMemories upserts a batch in one call", async () => { + const stale = "01900000-0000-7000-8000-0000000000b1"; + const fresh = "01900000-0000-7000-8000-0000000000b2"; + await db.batchCreateMemories(FULL, [ + { id: stale, tree: "work.batch", content: "old", meta: { v: "1" } }, + { id: fresh, tree: "work.batch", content: "current", meta: { v: "2" } }, + ]); + + const rows = await db.batchCreateMemories( + FULL, + [ + { id: stale, tree: "work.batch", content: "new", meta: { v: "2" } }, + { id: fresh, tree: "work.batch", content: "untouched", meta: { v: "2" } }, + { tree: "work.batch", content: "generated id" }, + ], + "v", + ); + const byId = new Map(rows.map((r) => [r.id, r.inserted])); + expect(rows).toHaveLength(2); // fresh skipped → absent + expect(byId.get(stale)).toBe(false); + expect((await db.getMemory(FULL, stale))?.content).toBe("new"); + expect((await db.getMemory(FULL, fresh))?.content).toBe("current"); + const generated = rows.find((r) => r.id !== stale); + expect(generated?.inserted).toBe(true); + expect((await db.getMemory(FULL, generated?.id as string))?.content).toBe( + "generated id", + ); +}); + test("access is enforced by the tree_access argument", async () => { // create requires write (>=2): read-only access is rejected await expect( diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts index 957eb87..1f9092d 100644 --- a/packages/engine/space/db.ts +++ b/packages/engine/space/db.ts @@ -31,6 +31,17 @@ export interface SpaceStore { treeAccess: TreeAccess, params: CreateMemoryParams, ): Promise<{ id: string; inserted: boolean } | null>; + /** + * Set-based createMemory for a whole batch: one statement, one round + * trip, same per-row conflict semantics. Returns one row per + * insert/replace — skipped rows are absent — and an explicit id repeated + * within the batch collapses to its first occurrence. Atomic. + */ + batchCreateMemories( + treeAccess: TreeAccess, + memories: CreateMemoryParams[], + replaceIfMetaDiffers?: string, + ): Promise>; getMemory(treeAccess: TreeAccess, id: string): Promise; patchMemory( treeAccess: TreeAccess, @@ -136,6 +147,27 @@ export function spaceStore(sql: Sql, schema: string): SpaceStore { return { id: row.id as string, inserted: Boolean(row.inserted) }; }, + async batchCreateMemories(treeAccess, memories, replaceIfMetaDiffers) { + if (memories.length === 0) return []; + // Parallel arrays aligned by position. Metas travel as ONE jsonb array + // via sql.json — a jsonb[] parameter would double-encode each element + // into a string scalar (see the jb() note above). + const rows = await sql` + select id, inserted from ${sch}.batch_create_memory( + ${jb(treeAccess)}, + ${memories.map((m) => m.id ?? null)}::uuid[], + ${memories.map((m) => m.tree)}::ltree[], + ${memories.map((m) => m.content)}::text[], + ${jb(memories.map((m) => m.meta ?? {}))}, + ${memories.map((m) => m.temporal ?? null)}::tstzrange[], + ${replaceIfMetaDiffers ?? null} + )`; + return rows.map((r) => ({ + id: r.id as string, + inserted: Boolean(r.inserted), + })); + }, + async getMemory(treeAccess, id) { const [row] = await sql` select id, tree::text as tree, meta, temporal::text as temporal, diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts index 251148b..264b387 100644 --- a/packages/server/rpc/memory/memory.ts +++ b/packages/server/rpc/memory/memory.ts @@ -199,13 +199,14 @@ async function memoryCreate( } /** - * memory.batchCreate — atomic across the batch. + * memory.batchCreate — atomic across the batch (one set-based statement, + * `batch_create_memory`). * * `ids` carries the inserted memories; `updatedIds` the existing rows - * rewritten via `replaceIfMetaDiffers` (conditional upsert in - * create_memory). A submitted explicit id in neither array was skipped — - * deterministic-id importers re-submit freely and classify the missing ids - * as already imported. + * rewritten via `replaceIfMetaDiffers` (conditional upsert). A submitted + * explicit id in neither array was skipped — deterministic-id importers + * re-submit freely and classify the missing ids as already imported. An id + * repeated within one batch collapses to its first occurrence. */ async function memoryBatchCreate( params: MemoryBatchCreateParams, @@ -215,25 +216,25 @@ async function memoryBatchCreate( const ctx = context as SpaceRpcContext; const { store, treeAccess } = ctx; - return await guard(() => - store.withTransaction(async (tx) => { - const ids: string[] = []; - const updatedIds: string[] = []; - for (const m of params.memories) { - const created = await tx.createMemory(treeAccess, { - id: m.id ?? undefined, - content: m.content, - meta: m.meta ?? undefined, - tree: inputTreePath(ctx, m.tree), - temporal: formatTemporal(m.temporal), - replaceIfMetaDiffers: params.replaceIfMetaDiffers ?? undefined, - }); - if (created === null) continue; - (created.inserted ? ids : updatedIds).push(created.id); - } - return { ids, updatedIds }; - }), + const rows = await guard(() => + store.batchCreateMemories( + treeAccess, + params.memories.map((m) => ({ + id: m.id ?? undefined, + content: m.content, + meta: m.meta ?? undefined, + tree: inputTreePath(ctx, m.tree), + temporal: formatTemporal(m.temporal), + })), + params.replaceIfMetaDiffers ?? undefined, + ), ); + const ids: string[] = []; + const updatedIds: string[] = []; + for (const r of rows) { + (r.inserted ? ids : updatedIds).push(r.id); + } + return { ids, updatedIds }; } /** memory.get */ diff --git a/packages/server/start.integration.test.ts b/packages/server/start.integration.test.ts index 2d95ca0..dca902b 100644 --- a/packages/server/start.integration.test.ts +++ b/packages/server/start.integration.test.ts @@ -156,7 +156,8 @@ test("re-migrates existing space schemas on boot", async () => { // The tampered function was in place just before boot… expect(tamperedDef).toContain("stale stand-in"); expect(tamperedDef).not.toContain("on conflict"); - // …and boot's space sweep re-applied the idempotent SQL over it. + // …and boot's space sweep re-applied the idempotent SQL over it + // (create_memory is the one-row wrapper delegating to batch_create_memory). expect(bootedDef).not.toContain("stale stand-in"); - expect(bootedDef).toContain("on conflict (id) do update"); + expect(bootedDef).toContain("batch_create_memory"); }); From 5ba9ae6d6a390af44c0256e44b637eed04d8311a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 20:02:56 +0200 Subject: [PATCH 138/156] fix(cli): scope `me claude init` transcript import to the current project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Init is per-project setup — the CLAUDE.md pointer and the git history import are already scoped to the repo — but its transcript step swept ALL of ~/.claude/projects, importing every project on the machine. The step now passes the repo root (or cwd outside a repo) as the project filter, so only sessions recorded in this project are backfilled; `me import claude` remains the machine-wide sweep. The temp-cwd filter is disabled for this scoped import: it exists to keep throwaway sessions out of bulk sweeps, but with the scope pinned to the directory the user is standing in it would only veto projects that happen to live under a temp dir. The e2e init tests now record their sessions in the init project's own directory (resolved via realpath — macOS tmpdir is a symlink and the filter compares against the resolved process.cwd()), and 8b additionally asserts a foreign project's session is NOT swept up. Co-Authored-By: Claude Fable 5 --- docs/cli/me-claude.md | 2 +- e2e/cli.e2e.test.ts | 147 ++++++++++++++++++++------------ packages/cli/commands/claude.ts | 19 ++++- 3 files changed, 108 insertions(+), 60 deletions(-) diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index 2ce630c..774d027 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -65,7 +65,7 @@ Setup is a list of independent steps. In an interactive terminal `init` presents | Step | Skip flag | What it does | |------|-----------|--------------| -| Import existing Claude Code sessions | `--skip-transcript-import` | Backfills sessions from `~/.claude/projects` — the same import as [`me import claude`](me-import.md#me-import-claude--codex--opencode). | +| Import this project's Claude Code sessions | `--skip-transcript-import` | Backfills sessions recorded in this project (cwd at/under the repo root, temp-dir projects included) from `~/.claude/projects`. For a machine-wide backfill across all projects, run [`me import claude`](me-import.md#me-import-claude--codex--opencode). | | Import git commit history | `--skip-git-import` | Imports the repo's full commit history — the same import as [`me import git`](me-import.md#me-import-git). Skipped automatically when the current directory is not inside a git repo. | | Add a memory pointer to CLAUDE.md | `--skip-claude-md` | Upserts a managed block into the project's CLAUDE.md naming the project tree (`share.projects.`), its `agent_sessions` and `git_history` nodes, and how to search them. Idempotent — re-runs replace the block in place. | diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index f9203a6..5216048 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -20,7 +20,14 @@ process.env.SPACE_SCHEMA_PREFIX = "metest_"; import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { authStore } from "@memory.build/auth"; @@ -453,47 +460,63 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( }); test("8b. `me claude init` backfills sessions and writes a CLAUDE.md pointer", async () => { - // `me claude init` is the one-shot setup command. Two steps: - // 1. import existing sessions (default --source ~/.claude/projects, - // which under this run's HOME=tmpHome is tmpHome/.claude/projects); + // `me claude init` is the one-shot setup command. Two steps exercised + // here: + // 1. import THIS project's existing sessions (sessions whose recorded + // cwd is at/under init's cwd — init is per-project setup; the + // machine-wide sweep is `me import claude`); // 2. record the project's memory location in the project's CLAUDE.md // (the project = init's cwd; not a git repo here → CLAUDE.md lands in // that dir, slug = its basename). const transcriptDir = join(tmpHome, ".claude", "projects", "init-proj"); await mkdir(transcriptDir, { recursive: true }); + // The project we run `init` in — a non-git temp dir with a known basename + // so the derived slug is predictable. CLAUDE.md will be written here, and + // the transcript's session records this dir as its cwd, so the session + // tree and the CLAUDE.md pointer name the same project. + const projectRoot = await mkdtemp(join(tmpdir(), "me-e2e-initcwd-")); + const projectDir = join(projectRoot, "initcwd"); + await mkdir(projectDir, { recursive: true }); + const sessionId = `init-${rand()}`; - // The transcript's own cwd "/work/init-proj" decides the session tree - // (independent of where `init` is invoked from). - const sessionCwd = "/work/init-proj"; - const tree = "share.projects.init_proj.agent_sessions"; - const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + const foreignId = `foreign-${rand()}`; + const tree = "share.projects.initcwd.agent_sessions"; + const mkMsg = ( + sid: string, + cwd: string, + i: number, + type: "user" | "assistant", + text: string, + ) => ({ type, - uuid: `${sessionId}-${type}-${i}`, + uuid: `${sid}-${type}-${i}`, timestamp: `2026-03-01T00:00:0${i}.000Z`, - sessionId, - cwd: sessionCwd, + sessionId: sid, + cwd, message: type === "user" ? { content: text } : { content: [{ type: "text", text }], model: "claude-x" }, }); - const lines = [ - mkMsg(0, "user", "init first question"), - mkMsg(1, "assistant", "init first answer"), - mkMsg(2, "user", "init second question"), - mkMsg(3, "assistant", "init second answer"), - ]; - await writeFile( - join(transcriptDir, `${sessionId}.jsonl`), - lines.map((l) => JSON.stringify(l)).join("\n"), - ); - - // The project we run `init` in — a non-git temp dir with a known basename - // so the derived slug is predictable. CLAUDE.md will be written here. - const projectRoot = await mkdtemp(join(tmpdir(), "me-e2e-initcwd-")); - const projectDir = join(projectRoot, "initcwd"); - await mkdir(projectDir, { recursive: true }); + const writeTranscript = (sid: string, cwd: string, prefix: string) => + writeFile( + join(transcriptDir, `${sid}.jsonl`), + [ + mkMsg(sid, cwd, 0, "user", `${prefix} first question`), + mkMsg(sid, cwd, 1, "assistant", `${prefix} first answer`), + mkMsg(sid, cwd, 2, "user", `${prefix} second question`), + mkMsg(sid, cwd, 3, "assistant", `${prefix} second answer`), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + // The recorded session cwd must be the REAL path (as Claude Code would + // record it): macOS tmpdir is a symlink (/var/folders → /private/var), + // and init filters against the resolved process.cwd(). + await writeTranscript(sessionId, await realpath(projectDir), "init"); + // A session from a DIFFERENT project must not be swept up by init. + await writeTranscript(foreignId, "/work/other-proj", "foreign"); // Pre-init: nothing captured, no CLAUDE.md. expect(await countBySession(sessionId)).toBe(0); @@ -502,9 +525,10 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( const init = await me(["claude", "init"], undefined, projectDir); expect(init.code, init.stderr).toBe(0); - // Step 1: the existing session was backfilled. + // Step 1: this project's session was backfilled; the foreign one wasn't. expect(await countBySession(sessionId)).toBe(4); expect(await countUnder(tree)).toBe(4); + expect(await countBySession(foreignId)).toBe(0); // Step 2: CLAUDE.md now points at this project's memories. const claudeMd = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); @@ -526,31 +550,10 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( test("8c. `me claude init` honors --skip-transcript-import / --skip-claude-md", async () => { // Non-interactive (piped) init runs every step except those turned off by // a --skip- flag. Verify each flag suppresses exactly its step. + // Each case gets its own project dir + a transcript recorded IN that dir + // (init's import is scoped to the project it runs in). const transcriptDir = join(tmpHome, ".claude", "projects", "skip-proj"); await mkdir(transcriptDir, { recursive: true }); - const sessionId = `skip-${rand()}`; - const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ - type, - uuid: `${sessionId}-${type}-${i}`, - timestamp: `2026-04-01T00:00:0${i}.000Z`, - sessionId, - cwd: "/work/skip-proj", - message: - type === "user" - ? { content: text } - : { content: [{ type: "text", text }], model: "claude-x" }, - }); - await writeFile( - join(transcriptDir, `${sessionId}.jsonl`), - [ - mkMsg(0, "user", "skip first question"), - mkMsg(1, "assistant", "skip first answer"), - mkMsg(2, "user", "skip second question"), - mkMsg(3, "assistant", "skip second answer"), - ] - .map((l) => JSON.stringify(l)) - .join("\n"), - ); const mkProject = async (name: string) => { const root = await mkdtemp(join(tmpdir(), "me-e2e-skip-")); @@ -558,27 +561,59 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( await mkdir(dir, { recursive: true }); return { root, dir }; }; + const writeTranscript = async (sid: string, cwd: string) => { + const mkMsg = (i: number, type: "user" | "assistant") => ({ + type, + uuid: `${sid}-${type}-${i}`, + timestamp: `2026-04-01T00:00:0${i}.000Z`, + sessionId: sid, + cwd, + message: + type === "user" + ? { content: `q${i}` } + : { + content: [{ type: "text", text: `a${i}` }], + model: "claude-x", + }, + }); + await writeFile( + join(transcriptDir, `${sid}.jsonl`), + [ + mkMsg(0, "user"), + mkMsg(1, "assistant"), + mkMsg(2, "user"), + mkMsg(3, "assistant"), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + }; - // --skip-transcript-import: CLAUDE.md is written, but nothing is imported. + // --skip-transcript-import: CLAUDE.md is written, but this project's + // session is NOT imported (it would have been without the flag). const a = await mkProject("skipimport"); + const sessionA = `skipa-${rand()}`; + await writeTranscript(sessionA, await realpath(a.dir)); const r1 = await me( ["claude", "init", "--skip-transcript-import"], undefined, a.dir, ); expect(r1.code, r1.stderr).toBe(0); - expect(await countBySession(sessionId)).toBe(0); + expect(await countBySession(sessionA)).toBe(0); expect(existsSync(join(a.dir, "CLAUDE.md"))).toBe(true); - // --skip-claude-md: the session imports, but no CLAUDE.md is written. + // --skip-claude-md: the project's session imports, but no CLAUDE.md. const b = await mkProject("skipclaudemd"); + const sessionB = `skipb-${rand()}`; + await writeTranscript(sessionB, await realpath(b.dir)); const r2 = await me( ["claude", "init", "--skip-claude-md"], undefined, b.dir, ); expect(r2.code, r2.stderr).toBe(0); - expect(await countBySession(sessionId)).toBe(4); + expect(await countBySession(sessionB)).toBe(4); expect(existsSync(join(b.dir, "CLAUDE.md"))).toBe(false); await rm(transcriptDir, { recursive: true, force: true }); diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 15918c2..7200e42 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -559,9 +559,22 @@ const INIT_STEPS: InitStep[] = [ id: "transcript-import", optionKey: "skipTranscriptImport", skipFlag: "--skip-transcript-import", - skipDescription: "do not import existing Claude Code sessions", - label: "Import existing Claude Code sessions", - run: ({ globalOpts }) => runAgentImport(claudeImporter, {}, globalOpts), + skipDescription: "do not import this project's Claude Code sessions", + label: "Import this project's Claude Code sessions", + // Init is per-project setup, so scope the backfill to sessions recorded + // in this repo (cwd at or under the repo root) — `me import claude` + // remains the machine-wide sweep. The temp-cwd filter exists to keep + // throwaway sessions out of bulk sweeps; with the scope pinned to the + // project the user is standing in, it would only veto projects that + // happen to live under a temp dir, so include them. + run: async ({ globalOpts }) => { + const { gitRoot } = await new SlugRegistry().resolve(process.cwd()); + await runAgentImport( + claudeImporter, + { project: gitRoot ?? process.cwd(), includeTempCwd: true }, + globalOpts, + ); + }, }, { id: "git-import", From dbe39b607a4796ab71f4f0698e512d62a233809c Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 20:08:55 +0200 Subject: [PATCH 139/156] fix(cli): announce each `me claude init` step before it runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first thing init printed was the transcript importer's progress spinner — raw scan counters and a .jsonl filename with no context. The init runner now prints each selected step's label (clack.log.step) before running it, so every step's output reads under a header: ◇ Import this project's Claude Code sessions ⠹ [8964 scanned, 7 processed] · … Text output only — --json/--yaml stay clean for parsing. Co-Authored-By: Claude Fable 5 --- packages/cli/commands/claude.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 7200e42..b741165 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -607,6 +607,7 @@ function createClaudeInitCommand(): Command { const globalOpts = cmdRef.optsWithGlobals(); const server = typeof globalOpts.server === "string" ? globalOpts.server : undefined; + const fmt = getOutputFormat(globalOpts); // Baseline = every step not explicitly turned off via its --skip-* flag. const baseline = INIT_STEPS.filter((s) => opts[s.optionKey] !== true); @@ -615,7 +616,7 @@ function createClaudeInitCommand(): Command { // with the baseline so the user can deselect steps. Otherwise run the // baseline as-is. const interactive = - getOutputFormat(globalOpts) === "text" && + fmt === "text" && Boolean(process.stdin.isTTY) && Boolean(process.stdout.isTTY); @@ -647,6 +648,10 @@ function createClaudeInitCommand(): Command { const ctx: InitStepContext = { globalOpts, server }; for (const step of selected) { + // Announce the step before its own output (progress spinners etc.) + // appears, so the counters have context. Structured output modes stay + // clean for parsing. + if (fmt === "text") clack.log.step(step.label); await step.run(ctx); } }); From 4dae3119f627cf744a9b6829364aa4d0da1ab7f2 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 20:18:22 +0200 Subject: [PATCH 140/156] perf(cli): prune claude transcript discovery by project directory name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A --project filter only applied per-session AFTER parsing: discovery still walked and parsed every transcript under ~/.claude/projects, so a scoped `me claude init` showed (and paid for) a machine-wide scan — 105k files on a busy machine for a 30-file project. Claude Code names each per-project directory after the session cwd with every non-alphanumeric character replaced by `-`, so discovery now skips directories that can't match the filter (exact encoded name or `-` prefix for descendant cwds). The encoding is lossy (`-` and `/` collide), so this is a prune only — kept files still pass the exact per-session cwd filter. Co-Authored-By: Claude Fable 5 --- packages/cli/importers/claude.test.ts | 45 +++++++++++++++++++++++++++ packages/cli/importers/claude.ts | 24 +++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/packages/cli/importers/claude.test.ts b/packages/cli/importers/claude.test.ts index 554d5eb..15bcf2c 100644 --- a/packages/cli/importers/claude.test.ts +++ b/packages/cli/importers/claude.test.ts @@ -9,6 +9,7 @@ import { describe, expect, test } from "bun:test"; import { join } from "node:path"; import { claudeImporter, + encodeProjectDir, sanitizeUserText, unwrapSdkReplayBundle, } from "./claude.ts"; @@ -59,6 +60,50 @@ function userTextsOf(messages: ConversationMessage[]): string[] { ); } +describe("encodeProjectDir", () => { + test("encodes a cwd the way Claude Code names project dirs", () => { + expect(encodeProjectDir("/Users/test/project")).toBe("-Users-test-project"); + // Trailing slashes are ignored; all non-alphanumerics become dashes. + expect(encodeProjectDir("/Users/x/my.app/")).toBe("-Users-x-my-app"); + expect(encodeProjectDir("/Users/x/my_app")).toBe("-Users-x-my-app"); + }); +}); + +describe("claude importer project-dir pruning", () => { + test("a matching --project keeps the project's directory", async () => { + const { sessions, stats } = await collect( + baseOptions({ projectFilter: "/Users/test/project" }), + ); + expect(stats.totalFiles).toBeGreaterThan(0); + expect(sessions.length).toBeGreaterThan(0); + }); + + test("an ancestor --project keeps descendant project directories", async () => { + const { sessions } = await collect( + baseOptions({ projectFilter: "/Users/test" }), + ); + expect(sessions.length).toBeGreaterThan(0); + }); + + test("a foreign --project never scans other projects' files", async () => { + const { sessions, stats } = await collect( + baseOptions({ projectFilter: "/Users/other/project" }), + ); + // Pruned at the directory level: zero files scanned, not scanned-then-skipped. + expect(stats.totalFiles).toBe(0); + expect(sessions).toEqual([]); + }); + + test("an encoded-prefix collision that is not a path ancestor is pruned", async () => { + // "/Users/test/proj" is a string prefix of the project but not an + // ancestor directory — the `${encoded}-` boundary must reject it. + const { stats } = await collect( + baseOptions({ projectFilter: "/Users/test/proj" }), + ); + expect(stats.totalFiles).toBe(0); + }); +}); + describe("claude importer", () => { test("skips sidechains by default", async () => { const { sessions, stats } = await collect(baseOptions()); diff --git a/packages/cli/importers/claude.ts b/packages/cli/importers/claude.ts index 96d5a5a..eddf944 100644 --- a/packages/cli/importers/claude.ts +++ b/packages/cli/importers/claude.ts @@ -14,7 +14,7 @@ */ import { promises as fs } from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { filterBySessionShape, recordSkip } from "./filters.ts"; import type { Importer } from "./index.ts"; import type { ProgressReporter } from "./progress.ts"; @@ -95,6 +95,18 @@ async function* discoverSessions( return; } + // With a --project filter, skip other projects' directories outright: the + // directory name encodes the session cwd, so most of the machine's + // transcripts never need to be opened. The encoding is lossy, so this only + // prunes — kept files still pass the exact per-session cwd filter below. + if (options.projectFilter) { + const encoded = encodeProjectDir(options.projectFilter); + projectDirs = projectDirs.filter((dir) => { + const name = basename(dir); + return name === encoded || name.startsWith(`${encoded}-`); + }); + } + for (const projectDir of projectDirs) { const files = await listJsonlFiles(projectDir); for (const file of files) { @@ -147,6 +159,16 @@ function countUserMessages(messages: ConversationMessage[]): number { } /** List immediate subdirectories of `path`. */ +/** + * Encode a cwd the way Claude Code names its per-project transcript + * directories: every non-alphanumeric character becomes `-` (e.g. + * /Users/x/my.app → -Users-x-my-app). Lossy (a literal `-` and a `/` encode + * identically), so matches are candidates, never authoritative. + */ +export function encodeProjectDir(cwd: string): string { + return cwd.replace(/\/+$/, "").replace(/[^a-zA-Z0-9]/g, "-"); +} + async function listSubdirs(path: string): Promise { const entries = await fs.readdir(path, { withFileTypes: true }); return entries From 24bf07021f3cc7da8d6ccc3b2de55bb2347230f4 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 20:51:47 +0200 Subject: [PATCH 141/156] docs: record the `me import` reorganization in DECISIONS_FOR_REVIEW Co-Authored-By: Claude Fable 5 --- DECISIONS_FOR_REVIEW.md | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md index 9da9fd1..16e8e87 100644 --- a/DECISIONS_FOR_REVIEW.md +++ b/DECISIONS_FOR_REVIEW.md @@ -297,3 +297,57 @@ the default, flip the `content_mode` userConfig default in **Status:** decided (per request); document the capture model when the docs are refreshed. + +--- + +## Imports reorganized under one `me import ` group; bare `me import ` removed + +**Date:** 2026-06-10 · **Area:** CLI command surface (`fc02772`) + +All imports now live under a single top-level umbrella group — `me import +memories | claude | codex | opencode | git` — one subcommand per **source**. +The pre-existing spellings remain registered as aliases built from the same +factories: `me memory import` ⇒ `me import memories`, and `me claude|codex| +opencode import` ⇒ `me import `. Two breaking consequences, both +deliberate: + +- **Bare `me import ` no longer parses.** `import` was previously the + auto-generated top-level alias of `me memory import`; the group now owns the + name, has **no default subcommand**, and its help text redirects old muscle + memory to `me import memories `. (Verified at the time: nothing in the + repo — tests, hooks, packs, docs — used the bare spelling.) +- `import` is excluded from the memory group's top-level auto-aliases + (`createMemoryAliasCommands`); every other memory subcommand (`me search`, + `me create`, …) still gets one. + +**Alternatives considered:** + +- *Per-source top-level groups* (`me git import`, matching `me claude import`): + rejected — every new source (gemini sessions, GitHub issues, Slack, …) would + cost a top-level command group, most containing only `import`. With the + umbrella, a new source is one subcommand; the integration groups (`me claude` + etc.) keep only genuine setup verbs (install/init/hook). +- *`files` as the umbrella's default subcommand* so bare `me import ` + keeps working (Commander `isDefault`): rejected — backward compatibility for + the bare spelling wasn't wanted, and a default reintroduces the ambiguity the + group exists to remove (`me import git` = the git source or a file named + `git`?). +- *Dropping the old spellings entirely*: rejected — `me memory import` is kept + for symmetry with `me memory export` (the data-plane inverse), and the + per-agent `import` aliases are kept since those groups exist anyway. +- Subcommand name `memories` over `files` — records are memories, not generic + files. + +**How to change it:** the group is assembled in +`packages/cli/commands/import-group.ts` (`createImportCommand`), from factories +that take a name parameter (`createMemoryImportCommand(name)` in +`commands/memory-import.ts`; `createClaudeImportCommand` etc. in +`commands/import.ts`); the alias exclusion is the `c.name() !== "import"` +filter in `commands/memory.ts:createMemoryAliasCommands`. To restore a bare +default, register the memories subcommand with Commander's `isDefault`; to drop +the legacy aliases, remove the `addCommand` calls in `memory.ts` / +`claude.ts` / `codex.ts` / `opencode.ts`. Docs: `docs/cli/me-import.md` is the +group page; the per-group pages note their alias status. + +**Status:** decided (user-directed, pre-release); recorded for rationale — +already reflected in `CLAUDE.md` and `docs/cli/`. From 4194a050fc106d6210b1cde82711273b7a4bff1b Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 20:57:49 +0200 Subject: [PATCH 142/156] feat(cli): point a freshly logged-in user at `me claude init` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful `me login` with an active space, print a "Next step" note telling the user to run `me claude init` at the root of a software development project. Skipped when no space is active — the existing create/select-a-space hints are the right next step there — and absent from --json/--yaml output. Co-Authored-By: Claude Fable 5 --- packages/cli/commands/login.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/cli/commands/login.ts b/packages/cli/commands/login.ts index 3b78853..1b2c6ec 100644 --- a/packages/cli/commands/login.ts +++ b/packages/cli/commands/login.ts @@ -159,6 +159,10 @@ export function createLoginCommand(): Command { clack.log.info(`Server: ${server}`); if (active) { clack.log.info(`Space: ${active.name} (${active.slug})`); + clack.note( + "Run 'me claude init' at the root of a software development\nproject to set up Claude Code memory for it.", + "Next step", + ); } else if (spaces.length === 0) { clack.log.info("No spaces yet. Run 'me space create '."); } else { From e269dd2f95c1c26695425b56b781cb3ec8ee9a32 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 21:15:45 +0200 Subject: [PATCH 143/156] feat(cli): `me claude init` offers to install the Claude Code plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Init wrote a CLAUDE.md pointing agents at the me_memory_search MCP tool and backfilled history — but without `me claude install` no MCP server or capture hooks exist, so init alone produced a half-working setup. A new first step runs the same install as `me claude install` (full plugin, user scope, login-session auth). Steps gain an optional `available` gate: a step resolving false is omitted entirely — no multiselect row, not in the non-interactive baseline. The install step hides itself when the `claude` binary is absent or `claude plugin list --json` already shows memory-engine@memory-engine (unparseable output counts as not installed — a wrong guess costs an idempotent re-install offer, never a missed one). The probe is skipped for steps opted out via their --skip flag in non-interactive runs, so `init --skip-plugin-install` never spawns `claude`. Also fixes the init e2e fixtures to mirror Claude Code's real on-disk layout: transcripts now live in encoded-cwd directory names (encodeProjectDir), which the project-scoped import has pruned by since 4dae311 — the literal "init-proj"/"skip-proj" fixture dirs were never scanned, a break that commit missed by not re-running e2e. The e2e init invocations pass --skip-plugin-install so a dev machine's claude never attempts a real marketplace install against the test HOME. Co-Authored-By: Claude Fable 5 --- docs/cli/me-claude.md | 1 + e2e/cli.e2e.test.ts | 76 +++++++++++++++++++--------- packages/cli/commands/claude.test.ts | 36 +++++++++++++ packages/cli/commands/claude.ts | 69 +++++++++++++++++++++++-- 4 files changed, 154 insertions(+), 28 deletions(-) create mode 100644 packages/cli/commands/claude.test.ts diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index 774d027..e4ed4ca 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -65,6 +65,7 @@ Setup is a list of independent steps. In an interactive terminal `init` presents | Step | Skip flag | What it does | |------|-----------|--------------| +| Install the Claude Code plugin | `--skip-plugin-install` | Runs the same install as [`me claude install`](#me-claude-install) (full plugin, `user` scope, login-session auth). Only offered when the `claude` binary is on PATH and `claude plugin list` doesn't already show the plugin — otherwise the step is hidden entirely. | | Import this project's Claude Code sessions | `--skip-transcript-import` | Backfills sessions recorded in this project (cwd at/under the repo root, temp-dir projects included) from `~/.claude/projects`. For a machine-wide backfill across all projects, run [`me import claude`](me-import.md#me-import-claude--codex--opencode). | | Import git commit history | `--skip-git-import` | Imports the repo's full commit history — the same import as [`me import git`](me-import.md#me-import-git). Skipped automatically when the current directory is not inside a git repo. | | Add a memory pointer to CLAUDE.md | `--skip-claude-md` | Upserts a managed block into the project's CLAUDE.md naming the project tree (`share.projects.`), its `agent_sessions` and `git_history` nodes, and how to search them. Idempotent — re-runs replace the block in place. | diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index 5216048..0b792cc 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -38,6 +38,7 @@ import { } from "@memory.build/database"; import type { EmbeddingConfig } from "@memory.build/embedding"; import type { Sql } from "postgres"; +import { encodeProjectDir } from "../packages/cli/importers/claude.ts"; import { connect, resolveTestDatabaseUrl, @@ -468,9 +469,6 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( // 2. record the project's memory location in the project's CLAUDE.md // (the project = init's cwd; not a git repo here → CLAUDE.md lands in // that dir, slug = its basename). - const transcriptDir = join(tmpHome, ".claude", "projects", "init-proj"); - await mkdir(transcriptDir, { recursive: true }); - // The project we run `init` in — a non-git temp dir with a known basename // so the derived slug is predictable. CLAUDE.md will be written here, and // the transcript's session records this dir as its cwd, so the session @@ -478,6 +476,10 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( const projectRoot = await mkdtemp(join(tmpdir(), "me-e2e-initcwd-")); const projectDir = join(projectRoot, "initcwd"); await mkdir(projectDir, { recursive: true }); + // The recorded session cwd must be the REAL path (as Claude Code would + // record it): macOS tmpdir is a symlink (/var/folders → /private/var), + // and init filters against the resolved process.cwd(). + const projectCwd = await realpath(projectDir); const sessionId = `init-${rand()}`; const foreignId = `foreign-${rand()}`; @@ -499,9 +501,18 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( ? { content: text } : { content: [{ type: "text", text }], model: "claude-x" }, }); - const writeTranscript = (sid: string, cwd: string, prefix: string) => - writeFile( - join(transcriptDir, `${sid}.jsonl`), + // Mirror Claude Code's on-disk layout: each transcript lives in a + // directory named after the session cwd (encoded) — the scoped import + // prunes by that name, so a literal fixture dir would never be scanned. + const writeTranscript = async ( + sid: string, + cwd: string, + prefix: string, + ) => { + const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); + await mkdir(dir, { recursive: true }); + await writeFile( + join(dir, `${sid}.jsonl`), [ mkMsg(sid, cwd, 0, "user", `${prefix} first question`), mkMsg(sid, cwd, 1, "assistant", `${prefix} first answer`), @@ -511,10 +522,8 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( .map((l) => JSON.stringify(l)) .join("\n"), ); - // The recorded session cwd must be the REAL path (as Claude Code would - // record it): macOS tmpdir is a symlink (/var/folders → /private/var), - // and init filters against the resolved process.cwd(). - await writeTranscript(sessionId, await realpath(projectDir), "init"); + }; + await writeTranscript(sessionId, projectCwd, "init"); // A session from a DIFFERENT project must not be swept up by init. await writeTranscript(foreignId, "/work/other-proj", "foreign"); @@ -522,7 +531,11 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(await countBySession(sessionId)).toBe(0); // Run `init` FROM the project dir so its cwd → slug → CLAUDE.md location. - const init = await me(["claude", "init"], undefined, projectDir); + const init = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + projectDir, + ); expect(init.code, init.stderr).toBe(0); // Step 1: this project's session was backfilled; the foreign one wasn't. @@ -538,12 +551,21 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(claudeMd).toContain("share.projects.initcwd.git_history"); // Re-running is idempotent: still exactly one managed block. - const init2 = await me(["claude", "init"], undefined, projectDir); + const init2 = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + projectDir, + ); expect(init2.code, init2.stderr).toBe(0); const claudeMd2 = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); expect(claudeMd2.split("memory-engine:start").length - 1).toBe(1); - await rm(transcriptDir, { recursive: true, force: true }); + for (const cwd of [projectCwd, "/work/other-proj"]) { + await rm(join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)), { + recursive: true, + force: true, + }); + } await rm(projectRoot, { recursive: true, force: true }); }); @@ -551,10 +573,9 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( // Non-interactive (piped) init runs every step except those turned off by // a --skip- flag. Verify each flag suppresses exactly its step. // Each case gets its own project dir + a transcript recorded IN that dir - // (init's import is scoped to the project it runs in). - const transcriptDir = join(tmpHome, ".claude", "projects", "skip-proj"); - await mkdir(transcriptDir, { recursive: true }); - + // (init's import is scoped to the project it runs in), stored under the + // Claude Code encoded-cwd directory layout the scoped import prunes by. + const transcriptDirs: string[] = []; const mkProject = async (name: string) => { const root = await mkdtemp(join(tmpdir(), "me-e2e-skip-")); const dir = join(root, name); @@ -576,8 +597,11 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( model: "claude-x", }, }); + const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); + transcriptDirs.push(dir); + await mkdir(dir, { recursive: true }); await writeFile( - join(transcriptDir, `${sid}.jsonl`), + join(dir, `${sid}.jsonl`), [ mkMsg(0, "user"), mkMsg(1, "assistant"), @@ -595,7 +619,7 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( const sessionA = `skipa-${rand()}`; await writeTranscript(sessionA, await realpath(a.dir)); const r1 = await me( - ["claude", "init", "--skip-transcript-import"], + ["claude", "init", "--skip-transcript-import", "--skip-plugin-install"], undefined, a.dir, ); @@ -608,7 +632,7 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( const sessionB = `skipb-${rand()}`; await writeTranscript(sessionB, await realpath(b.dir)); const r2 = await me( - ["claude", "init", "--skip-claude-md"], + ["claude", "init", "--skip-claude-md", "--skip-plugin-install"], undefined, b.dir, ); @@ -616,7 +640,9 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( expect(await countBySession(sessionB)).toBe(4); expect(existsSync(join(b.dir, "CLAUDE.md"))).toBe(false); - await rm(transcriptDir, { recursive: true, force: true }); + for (const dir of transcriptDirs) { + await rm(dir, { recursive: true, force: true }); + } await rm(a.root, { recursive: true, force: true }); await rm(b.root, { recursive: true, force: true }); }); @@ -771,7 +797,7 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( // --skip-git-import: no commit memories. const skipped = await me( - ["claude", "init", "--skip-git-import"], + ["claude", "init", "--skip-git-import", "--skip-plugin-install"], undefined, repo, ); @@ -780,7 +806,11 @@ describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( // Plain init (non-interactive baseline) imports the repo's history and // the CLAUDE.md pointer names the git_history node. - const init = await me(["claude", "init"], undefined, repo); + const init = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + repo, + ); expect(init.code, init.stderr).toBe(0); expect(await countUnder(tree)).toBe(1); const claudeMd = await readFile(join(repo, "CLAUDE.md"), "utf8"); diff --git a/packages/cli/commands/claude.test.ts b/packages/cli/commands/claude.test.ts new file mode 100644 index 0000000..e323168 --- /dev/null +++ b/packages/cli/commands/claude.test.ts @@ -0,0 +1,36 @@ +/** + * Tests for `me claude` helpers. + */ +import { describe, expect, test } from "bun:test"; +import { pluginListShowsInstalled } from "./claude.ts"; + +describe("pluginListShowsInstalled", () => { + test("finds the plugin by its id in `claude plugin list --json` output", () => { + const out = JSON.stringify([ + { id: "superpowers@superpowers-marketplace", enabled: true }, + { id: "memory-engine@memory-engine", version: "0.1.0", enabled: true }, + ]); + expect(pluginListShowsInstalled(out)).toBe(true); + }); + + test("a disabled install still counts as installed", () => { + const out = JSON.stringify([ + { id: "memory-engine@memory-engine", enabled: false }, + ]); + expect(pluginListShowsInstalled(out)).toBe(true); + }); + + test("other plugins do not match", () => { + const out = JSON.stringify([ + { id: "memory-engine-fork@somewhere", enabled: true }, + ]); + expect(pluginListShowsInstalled(out)).toBe(false); + }); + + test("empty list and unparseable output count as not installed", () => { + expect(pluginListShowsInstalled("[]")).toBe(false); + expect(pluginListShowsInstalled("")).toBe(false); + expect(pluginListShowsInstalled("Installed plugins:\n ❯ …")).toBe(false); + expect(pluginListShowsInstalled('{"not": "an array"}')).toBe(false); + }); +}); diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index b741165..5f62cc7 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -550,11 +550,59 @@ interface InitStep { skipDescription: string; /** Multiselect row label. */ label: string; + /** + * Optional availability gate: a step that resolves false is omitted + * entirely — no multiselect row and not part of the non-interactive + * baseline. Absent means always available. + */ + available?: () => Promise; /** Perform the step. */ run: (ctx: InitStepContext) => Promise; } +/** + * Parse `claude plugin list --json` output and report whether the Memory + * Engine plugin is installed. Exported for tests. Unparseable output counts + * as not-installed — the wrong guess costs an idempotent re-install offer, + * never a missed install. + */ +export function pluginListShowsInstalled(stdout: string): boolean { + try { + const plugins = JSON.parse(stdout); + if (!Array.isArray(plugins)) return false; + return plugins.some((p) => (p as { id?: unknown }).id === PLUGIN_REF); + } catch { + return false; + } +} + +/** + * Availability of the plugin-install init step: offered only when the + * `claude` binary exists and the plugin is not already installed. + */ +async function pluginInstallAvailable(): Promise { + if (Bun.which("claude") === null) return false; + const { exitCode, stdout } = await runCommand([ + "claude", + "plugin", + "list", + "--json", + ]); + if (exitCode !== 0) return true; // can't tell → offer the install + return !pluginListShowsInstalled(stdout); +} + const INIT_STEPS: InitStep[] = [ + { + id: "plugin-install", + optionKey: "skipPluginInstall", + skipFlag: "--skip-plugin-install", + skipDescription: "do not install the Claude Code plugin", + label: "Install the Claude Code plugin (hooks + slash commands + MCP)", + // Hidden when Claude Code is absent or the plugin is already installed. + available: pluginInstallAvailable, + run: ({ server }) => runClaudePluginInstall({ server, scope: "user" }), + }, { id: "transcript-import", optionKey: "skipTranscriptImport", @@ -609,9 +657,6 @@ function createClaudeInitCommand(): Command { typeof globalOpts.server === "string" ? globalOpts.server : undefined; const fmt = getOutputFormat(globalOpts); - // Baseline = every step not explicitly turned off via its --skip-* flag. - const baseline = INIT_STEPS.filter((s) => opts[s.optionKey] !== true); - // Interactive (a TTY with text output): present a multiselect pre-checked // with the baseline so the user can deselect steps. Otherwise run the // baseline as-is. @@ -620,11 +665,25 @@ function createClaudeInitCommand(): Command { Boolean(process.stdin.isTTY) && Boolean(process.stdout.isTTY); + // Steps available in this environment (e.g. plugin-install hides itself + // when Claude Code is absent or the plugin is already installed). The + // probe is skipped for steps already opted out non-interactively, so a + // `--skip-` run never pays for that step's availability check. + const candidates: InitStep[] = []; + for (const step of INIT_STEPS) { + if (!interactive && opts[step.optionKey] === true) continue; + if (step.available && !(await step.available())) continue; + candidates.push(step); + } + + // Baseline = every available step not turned off via its --skip-* flag. + const baseline = candidates.filter((s) => opts[s.optionKey] !== true); + let selectedIds: string[]; if (interactive) { const picked = await clack.multiselect({ message: `Setup steps to run ${DIM}(all selected by default — ↑/↓ move, space to toggle off/on, enter to confirm)${DIM_OFF}`, - options: INIT_STEPS.map((s) => ({ + options: candidates.map((s) => ({ value: s.id, label: s.label, })), @@ -640,7 +699,7 @@ function createClaudeInitCommand(): Command { selectedIds = baseline.map((s) => s.id); } - const selected = INIT_STEPS.filter((s) => selectedIds.includes(s.id)); + const selected = candidates.filter((s) => selectedIds.includes(s.id)); if (selected.length === 0) { clack.log.info("No setup steps selected — nothing to do."); return; From 8f0c056b5f4b6043fab41e028258b5f18e5e5657 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 21:36:49 +0200 Subject: [PATCH 144/156] docs: add a quick-start-against-dev flow to DEVELOPMENT.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clone → ./bun install + install:local → login against the dev server → `me claude install --dev`. Login precedes the plugin install (it needs the session and stored server URL), --dev runs from inside the repo, and the note explains that a dev-installed plugin keeps `me claude init` from offering the published one over it. Co-Authored-By: Claude Fable 5 --- DEVELOPMENT.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 2abb097..7cade02 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -7,11 +7,36 @@ ## Quick Start against dev +### 1. Clone the repo + ```bash +git clone git@github.com:timescale/memory-engine.git +cd memory-engine +``` + +### 2. Install + +```bash +./bun install ./bun run install:local +``` + +### 3. Log in and install the Claude Code plugin + +```bash me --server https://me.dev-us-east-1.ops.dev.timescale.com login me claude install --dev -``` +``` + +Login must come first: `me claude install` needs your session and the +stored server URL. `--dev` installs the Claude Code plugin from your local +checkout (run it from inside the repo) instead of the published +marketplace — with it installed, `me claude init` won't offer to install +the published plugin over it. + +After that follow the instructions from login. The next step will probably +be `me claude init` in whatever project you are working in. Don't test on +memory-engine itself as that can be confusing for the model. ## Quick Start From cd0534386b2357a5550fffc1ca5f6ccfa53412fc Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Wed, 10 Jun 2026 22:01:08 +0200 Subject: [PATCH 145/156] fix(test): scope the boot test's SPACE_SCHEMA_PREFIX to its own suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI runs every integration file in ONE bun process (`find … | xargs bun test`), so start.integration.test.ts's module-scope `process.env.SPACE_SCHEMA_PREFIX = "metest_"` leaked into every suite that ran after it: their provisionUser created metest_ schemas while the tests query the hardcoded me_ — 20 handler tests failing with `schema "me_…" does not exist`. Local runs were green because `./bun run check` passes --parallel, which isolates files per process. The prefix is now set in beforeAll and restored in afterAll (the slug.test.ts save/restore pattern); the e2e suite keeps its module-scope assignment since it runs in its own invocation. Verified with the exact CI command shape: all 17 integration files in one process, 225 pass. Co-Authored-By: Claude Fable 5 --- packages/server/start.integration.test.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/server/start.integration.test.ts b/packages/server/start.integration.test.ts index dca902b..e2c63bb 100644 --- a/packages/server/start.integration.test.ts +++ b/packages/server/start.integration.test.ts @@ -7,10 +7,6 @@ // TEST_DATABASE_URL="$(ghost connect testing_me)" \ // bun test --timeout 30000 packages/server/start.integration.test.ts -// Space schemas created by this test land under metest_ (not production -// me_) so leftovers are reclaimable by name. Set before anything reads it. -process.env.SPACE_SCHEMA_PREFIX = "metest_"; - import { afterAll, beforeAll, expect, test } from "bun:test"; import { bootstrapSpaceDatabase, @@ -49,6 +45,7 @@ let coreSchema: string; let spaceSchema: string; let tamperedDef: string; let bootedDef: string; +let prevSchemaPrefix: string | undefined; /** Current definition of a space schema's create_memory function. */ async function createMemoryDef(schema: string): Promise { @@ -61,6 +58,14 @@ async function createMemoryDef(schema: string): Promise { } beforeAll(async () => { + // Space schemas created by this suite land under metest_ (not + // production me_) so leftovers are reclaimable by name. Scoped to + // this suite and restored in afterAll: CI runs every integration file in + // ONE bun process (`find … | xargs bun test`), so a module-scope + // assignment would leak the prefix into suites that expect me_. + prevSchemaPrefix = process.env.SPACE_SCHEMA_PREFIX; + process.env.SPACE_SCHEMA_PREFIX = "metest_"; + authSchema = `auth_test_${rand()}`; coreSchema = `core_test_${rand()}`; sql = postgres(URL, { onnotice: () => {} }); @@ -125,6 +130,11 @@ afterAll(async () => { await sql.unsafe(`drop schema if exists ${authSchema} cascade`); await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); await sql.end(); + if (prevSchemaPrefix === undefined) { + delete process.env.SPACE_SCHEMA_PREFIX; + } else { + process.env.SPACE_SCHEMA_PREFIX = prevSchemaPrefix; + } }); test("boots on a random port and serves /health", async () => { From c4838782514c555d7762942a5a399c62e16d2a38 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 09:45:37 +0200 Subject: [PATCH 146/156] ci: run tests through the shared scripts (test:unit / test:db) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's bare `find … | xargs bun test` ran every test file in one process while local runs use --parallel (per-file isolation) — the divergence that let a module-scope env leak pass locally and break CI (cd05343). CI now calls package.json scripts so the command shape has one source of truth: the integration job reuses the existing test:db (whose schema cleaner is a no-op against the fresh CI container), and a new test:unit mirrors it for non-integration files. Both carry --parallel, matching the local process model. Deliberately NOT `./bun run check`: check is the dev loop — its `lint --write` would auto-fix violations in the runner instead of failing, and it merges everything into one job, putting the Docker Postgres build on the critical path of every lint error. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 10 ++++++++-- package.json | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eda52a..e4b36da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,9 @@ jobs: - name: Typecheck run: ./bun run typecheck - name: Test (unit) - run: find packages -name '*.test.ts' ! -name '*.integration.test.ts' -print0 | xargs -0 ./bun test + # Same script (and process model — --parallel) as local runs, so CI + # can't diverge from what developers verify against. + run: ./bun run test:unit integration: runs-on: ubuntu-latest @@ -51,6 +53,10 @@ jobs: sleep 1 done - name: Test (integration) - run: find packages -name '*.integration.test.ts' -print0 | xargs -0 ./bun test + # Same script (and process model — --parallel) as local runs; the + # suites default to postgresql://postgres@127.0.0.1:5432/postgres, + # which is the container above. The schema cleaner that test:db runs + # first is a no-op against the fresh container. + run: ./bun run test:db - name: Stop Postgres run: docker stop me-postgres diff --git a/package.json b/package.json index 003c1d2..861be99 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "test:db:clean": "./bun scripts/clean-test-schemas.ts", "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", "test:e2e": "./bun run test:db:clean && ./bun test --timeout 60000 ./e2e", + "test:unit": "find packages -name '*.test.ts' ! -name '*.integration.test.ts' -print0 | xargs -0 ./bun test --parallel=2", "typecheck": "tsc --noEmit" }, "devDependencies": { From 47b92ca932c08e01e02da07299e66df030516aa4 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 09:51:41 +0200 Subject: [PATCH 147/156] fix(db): serialize extension creation behind one database-wide lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaced by the previous commit: running the integration suites with --parallel against the FRESH CI container failed with a unique_violation from `create extension`. Extensions are database-global, but each migrator (auth, core, the space bootstrap) serializes only against its own advisory key — two different migrators racing on a fresh database both pass ensureExtension's existence check and the loser dies on the pg_extension catalog insert. Never seen against ghost or an established deployment because the extensions already exist there; two server replicas booting against a fresh database could hit the same race. ensureExtension now takes a single database-wide advisory xact lock ("memory:extensions") before the existence check. The lock is the only one shared across migrators and is always acquired last, so no ordering cycle; it holds until the winner's commit makes the extension visible, at which point the loser's check sees it and no-ops. Re-acquisition within one migrator's transaction is immediate. Co-Authored-By: Claude Fable 5 --- packages/database/migrate/kit.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/database/migrate/kit.ts b/packages/database/migrate/kit.ts index 90f233f..138b6d7 100644 --- a/packages/database/migrate/kit.ts +++ b/packages/database/migrate/kit.ts @@ -102,6 +102,18 @@ export async function ensureExtension( name: string, minVersion: string, ): Promise { + // Extensions are database-global, but each migrator (auth, core, the + // space bootstrap) serializes only against its own advisory key — two + // DIFFERENT migrators racing on a fresh database both pass the existence + // check below and the loser's `create extension` dies with a + // unique_violation (seen with parallel integration suites on a fresh CI + // container). One database-wide lock serializes every extension ensure; + // transaction-scoped, so a loser proceeds only after the winner's commit + // made the extension visible. Re-acquiring within the same transaction + // (one lock per extension in a migrator's loop) is immediate. + const [key1, key2] = advisoryLockKey("memory:extensions"); + await tx`select pg_advisory_xact_lock(${key1}, ${key2})`; + const [installed] = await tx` select x.extversion, n.nspname from pg_extension x From 0e06eea6a99f1452a4de4dee4646b1488f0da9fa Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 14:24:14 +0200 Subject: [PATCH 148/156] ci: TEST_CI disables test skips; lighter default `check` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI now sets TEST_CI=1 in both jobs: conditional describe.skipIf gates include !process.env.TEST_CI, so in CI every gated suite RUNS and missing prerequisites fail loudly as test errors instead of skipping silently (the e2e suite had been "passing" CI for its entire life without executing). The integration job gains a test:e2e step (real CLI + real in-process server + real OpenAI embeddings against the CI Postgres container) and both jobs receive the OPENAI_API_KEY secret — the unit job runs the live OpenAI embedding suite. Requires the OPENAI_API_KEY repo secret; until it exists, CI fails on those suites (by design: silence is an error). Fork PRs receive no secrets and fail the same way. The Ollama embedding suite is deleted rather than exempted: it needs a live local Ollama (dev-only provider, never deployed) and would be the lone skip CI tolerates. The shared embedding code keeps live coverage through the OpenAI suite; test the Ollama provider manually against a local Ollama if touching it. Locally the rigor inverts: `check` is now the fast inner loop (typecheck + lint + unit tests, no database, ~13s) and `check:full` carries the old everything-chain — including actually running e2e, which the old check silently skipped without TEST_DATABASE_URL (it now defaults to ghost). CLAUDE.md documents the convention: new skipIf gates must include !process.env.TEST_CI. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 18 + CLAUDE.md | 23 +- DEVELOPMENT.md | 5 +- e2e/cli.e2e.test.ts | 1761 +++++++++++++-------------- package.json | 3 +- packages/embedding/generate.test.ts | 78 +- packs/README.md | 2 +- 7 files changed, 920 insertions(+), 970 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4b36da..7eb79c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,13 @@ on: jobs: ci: runs-on: ubuntu-latest + env: + # TEST_CI disables conditional test skips: every gated suite runs, and + # missing prerequisites fail loudly instead of skipping silently. The + # OpenAI key drives the live embedding suite; a fork PR (no secrets) + # therefore fails rather than silently testing less. + TEST_CI: "1" + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} steps: - uses: actions/checkout@v6 @@ -31,6 +38,13 @@ jobs: integration: runs-on: ubuntu-latest + env: + # See the ci job for TEST_CI semantics. The e2e suite needs the database + # URL (the container below) and a real OpenAI key for its embedding + # worker (a few thousand text-embedding-3-small tokens per run). + TEST_CI: "1" + TEST_DATABASE_URL: postgresql://postgres@127.0.0.1:5432/postgres + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} steps: - uses: actions/checkout@v6 @@ -58,5 +72,9 @@ jobs: # which is the container above. The schema cleaner that test:db runs # first is a no-op against the fresh container. run: ./bun run test:db + - name: Test (e2e) + # Real CLI subprocesses against a real in-process server and the + # container above, with real OpenAI embeddings. + run: ./bun run test:e2e - name: Stop Postgres run: docker stop me-postgres diff --git a/CLAUDE.md b/CLAUDE.md index 15f2c5e..36bab2b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,21 +86,28 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): # fast, for iterating on one file: ./bun test packages/cli/mcp/install.test.ts -# Full suite (unit + integration). `test` and `check` default TEST_DATABASE_URL -# to the ghost instance and run files in parallel (--parallel=2) with a 30s -# timeout; set TEST_DATABASE_URL (e.g. a local Postgres) to override. +# Full suite (unit + integration). `test` defaults TEST_DATABASE_URL to the +# ghost instance and runs files in parallel (--parallel=2) with a 30s timeout; +# set TEST_DATABASE_URL (e.g. a local Postgres) to override. ./bun run test -# Shorthand for all checks (typecheck + lint + test → ghost) +# Fast inner loop (typecheck + lint + unit tests; no database, ~15s) ./bun run check + +# Everything: check + full suite vs ghost + the e2e suite (~5 min) +./bun run check:full ``` -**Important**: After making code changes, always run `./bun run check` (it runs -the full suite against ghost, ~4 min; for a faster inner loop set -`TEST_DATABASE_URL` to a local Postgres). +**Important**: After making code changes, run `./bun run check` (fast, no DB). +Before committing, run `./bun run check:full` — the full suite against ghost +plus the e2e suite. CI is the strict gate: it runs every suite with `TEST_CI=1`, +which disables conditional skips — any new `describe.skipIf` gate **must** +include `!process.env.TEST_CI` in its condition (pattern: +`packages/embedding/generate.test.ts`, `e2e/cli.e2e.test.ts`) so CI never +silently skips it. > `packages/web` and `packages/docs-site` are excluded from the root typecheck -> (they have their own); `./bun run check` does not cover them. +> (they have their own); `check`/`check:full` do not cover them. ### Database integration tests diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7cade02..f5a1808 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -209,8 +209,9 @@ After login, the server URL is stored as the default in `~/.config/me/credential | `./bun run pg` | Build and start PostgreSQL in Docker | | `./bun run pg:rm` | Stop and remove the PostgreSQL container | | `./bun run psql` | Connect to PostgreSQL with psql | -| `./bun run test` | Run tests | -| `./bun run check` | Format + lint + typecheck | +| `./bun run test` | Run all package tests (unit + integration, vs ghost by default) | +| `./bun run check` | Fast inner loop: typecheck + lint + unit tests (no database) | +| `./bun run check:full` | Everything: check + full suite vs ghost + e2e | | `./bun run build` | Compile CLI binary (current platform) | | `./bun run build:all` | Cross-compile CLI for all platforms | | `./bun run install:local` | Build and install local CLI binary to your PATH | diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index 0b792cc..b484dc9 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -67,927 +67,920 @@ let spaceSlug: string; let token: string; let tmpHome: string; -describe.skipIf(!OPENAI_KEY || !process.env.TEST_DATABASE_URL)( - "cli e2e", - () => { - beforeAll(async () => { - sql = connect(); - authSchema = `auth_test_${rand()}`; - coreSchema = `core_test_${rand()}`; - await bootstrapSpaceDatabase(sql); - await migrateAuth(sql, { schema: authSchema }); - await migrateCore(sql, { schema: coreSchema }); - - // Provision the user (and its default space) BEFORE booting the server, so - // the worker discovers the space at startup — no rediscovery lag for the - // initial space. - const provisioned = await provisionUser( - sql, - { auth: authSchema, core: coreSchema }, - { - email: "e2e@example.test", - name: "E2E", - provider: "github", - accountId: `e2e-${rand()}`, - emailVerified: true, - }, - ); - spaceSlug = provisioned.spaceSlug; - ({ token } = await authStore(sql, authSchema).createSession( - provisioned.userId, - )); - - const embeddingConfig: EmbeddingConfig = { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - // OPENAI_KEY is non-null here (describe.skipIf guards it). - apiKey: OPENAI_KEY as string, - options: {}, - }; - - srv = await startServer({ - port: 0, - databaseUrl: resolveTestDatabaseUrl(), - apiBaseUrl: "http://localhost", // OAuth callbacks unused (token injection) - authSchema, - coreSchema, - migrate: false, // harness already migrated - enableCleanupCron: false, - workerCount: 1, - workerIdleDelayMs: 250, // poll the embed queue fast - workerRefreshIntervalMs: 500, // discover new spaces fast - embeddingConfig, - }); - - tmpHome = await mkdtemp(join(tmpdir(), "me-e2e-")); +// TEST_CI disables the conditional skip: in CI this suite always runs +// (missing env fails loudly as test errors, never as a silent skip). +describe.skipIf( + !process.env.TEST_CI && (!OPENAI_KEY || !process.env.TEST_DATABASE_URL), +)("cli e2e", () => { + beforeAll(async () => { + sql = connect(); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); + + // Provision the user (and its default space) BEFORE booting the server, so + // the worker discovers the space at startup — no rediscovery lag for the + // initial space. + const provisioned = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: "e2e@example.test", + name: "E2E", + provider: "github", + accountId: `e2e-${rand()}`, + emailVerified: true, + }, + ); + spaceSlug = provisioned.spaceSlug; + ({ token } = await authStore(sql, authSchema).createSession( + provisioned.userId, + )); + + const embeddingConfig: EmbeddingConfig = { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + // OPENAI_KEY is non-null here (describe.skipIf guards it). + apiKey: OPENAI_KEY as string, + options: {}, + }; + + srv = await startServer({ + port: 0, + databaseUrl: resolveTestDatabaseUrl(), + apiBaseUrl: "http://localhost", // OAuth callbacks unused (token injection) + authSchema, + coreSchema, + migrate: false, // harness already migrated + enableCleanupCron: false, + workerCount: 1, + workerIdleDelayMs: 250, // poll the embed queue fast + workerRefreshIntervalMs: 500, // discover new spaces fast + embeddingConfig, }); - afterAll(async () => { - await srv?.stop(); - // Drop the space schemas this run created (enumerating core.space covers - // CLI-created spaces too), then the auth/core test schemas. - if (sql && coreSchema) { - const spaces = await sql.unsafe(`select slug from ${coreSchema}.space`); - for (const row of spaces) { - await sql.unsafe(`drop schema if exists metest_${row.slug} cascade`); - } - await sql.unsafe(`drop schema if exists ${authSchema} cascade`); - await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); - await sql.end(); + tmpHome = await mkdtemp(join(tmpdir(), "me-e2e-")); + }); + + afterAll(async () => { + await srv?.stop(); + // Drop the space schemas this run created (enumerating core.space covers + // CLI-created spaces too), then the auth/core test schemas. + if (sql && coreSchema) { + const spaces = await sql.unsafe(`select slug from ${coreSchema}.space`); + for (const row of spaces) { + await sql.unsafe(`drop schema if exists metest_${row.slug} cascade`); } - if (tmpHome) await rm(tmpHome, { recursive: true, force: true }); - }); - - // ------------------------------------------------------------------------- - // CLI subprocess helpers - // ------------------------------------------------------------------------- - - function cliEnv( - extra: Record = {}, - ): Record { - const env = { ...process.env } as Record; - // Curate: drop any ambient ME_* so the dev's shell can't leak in. - for (const k of [ - "ME_API_KEY", - "ME_SERVER", - "ME_SPACE", - "ME_SESSION_TOKEN", - ]) { - delete env[k]; - } - return { - ...env, - HOME: tmpHome, - XDG_CONFIG_HOME: join(tmpHome, ".config"), - ME_NO_KEYCHAIN: "1", - ME_SERVER: srv.url, - ME_SESSION_TOKEN: token, - ME_SPACE: spaceSlug, - ...extra, - }; - } - - async function me( - args: string[], - extraEnv?: Record, - cwd?: string, - ): Promise<{ stdout: string; stderr: string; code: number }> { - const proc = Bun.spawn([process.execPath, CLI, ...args], { - env: cliEnv(extraEnv), - stdout: "pipe", - stderr: "pipe", - ...(cwd ? { cwd } : {}), - }); - const [stdout, stderr] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - ]); - const code = await proc.exited; - return { stdout, stderr, code }; - } - - // Like `me`, but pipes `input` to the process's stdin (for `me claude hook`, - // which reads the event JSON from stdin). - async function meStdin( - args: string[], - input: string, - extraEnv?: Record, - ): Promise<{ stdout: string; stderr: string; code: number }> { - const proc = Bun.spawn([process.execPath, CLI, ...args], { - env: cliEnv(extraEnv), - stdin: new TextEncoder().encode(input), - stdout: "pipe", - stderr: "pipe", - }); - const [stdout, stderr] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - ]); - const code = await proc.exited; - return { stdout, stderr, code }; + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); } - - // Count memories under a tree in this run's space schema. - async function countUnder(treePrefix: string): Promise { - const [row] = await sql.unsafe( - `select count(*)::int as n from metest_${spaceSlug}.memory - where tree <@ $1::ltree`, - [treePrefix], - ); - return (row?.n as number) ?? 0; - } - - // Count memories captured from a given source session id. - async function countBySession(sessionId: string): Promise { - const [row] = await sql.unsafe( - `select count(*)::int as n from metest_${spaceSlug}.memory - where meta->>'source_session_id' = $1`, - [sessionId], - ); - return (row?.n as number) ?? 0; + if (tmpHome) await rm(tmpHome, { recursive: true, force: true }); + }); + + // ------------------------------------------------------------------------- + // CLI subprocess helpers + // ------------------------------------------------------------------------- + + function cliEnv(extra: Record = {}): Record { + const env = { ...process.env } as Record; + // Curate: drop any ambient ME_* so the dev's shell can't leak in. + for (const k of [ + "ME_API_KEY", + "ME_SERVER", + "ME_SPACE", + "ME_SESSION_TOKEN", + ]) { + delete env[k]; } - - // Parse the --json stdout of a `me` invocation, asserting success. - async function meJson( - args: string[], - extraEnv?: Record, - ): Promise { - const r = await me([...args, "--json"], extraEnv); - expect( - r.code, - `expected exit 0 for \`me ${args.join(" ")}\`\nstdout: ${r.stdout}\nstderr: ${r.stderr}`, - ).toBe(0); - return JSON.parse(r.stdout) as T; - } - - // Poll the space schema until N memories have a non-null embedding. - async function waitForEmbeddings(count: number, timeoutMs = 30000) { - const schema = `metest_${spaceSlug}`; - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const [row] = await sql.unsafe( - `select count(*)::int as n from ${schema}.memory where embedding is not null`, - ); - if ((row?.n ?? 0) >= count) return; - await Bun.sleep(250); - } - throw new Error(`timed out waiting for ${count} embeddings`); - } - - // ------------------------------------------------------------------------- - // Core scenarios - // ------------------------------------------------------------------------- - - test("1. whoami reports the provisioned identity", async () => { - const r = await me(["whoami"]); - expect(r.code).toBe(0); - expect(r.stdout).toContain("e2e@example.test"); - }); - - test("2. create + tree round-trip (share namespace)", async () => { - const created = await meJson<{ id: string; tree?: string }>([ - "create", - "the quick brown fox jumps over the lazy dog", - "--tree", - "share", - ]); - expect(created.id).toBeTruthy(); - - const r = await me(["memory", "tree"]); - expect(r.code).toBe(0); - expect(r.stdout.toLowerCase()).toContain("share"); - }); - - test("3. fulltext (BM25) search finds the memory", async () => { - const res = await meJson<{ - total: number; - results: { id: string; content: string }[]; - }>(["search", "--fulltext", "fox"]); - expect(res.total).toBeGreaterThan(0); - expect( - res.results.some((m) => m.content.includes("quick brown fox")), - ).toBe(true); - }); - - test("4. semantic search ranks a paraphrase near the top", async () => { - // Seed a few more memories to make ranking meaningful. - const seed = (text: string) => - meJson(["create", text, "--tree", "share"]); - await seed("a dog chased a cat across the yard"); - await seed("the stock market fell sharply on Tuesday"); - await seed("photosynthesis converts sunlight into energy"); - - // 4 created so far in `share` (1 from scenario 2 + 3 here). Wait for the - // worker to embed them. - await waitForEmbeddings(4); - - const res = await meJson<{ - results: { id: string; content: string }[]; - }>(["search", "--semantic", "wild canine leaps over a sleepy hound"]); - // Recall-based: the fox/dog memories should surface near the top, not the - // stock-market or photosynthesis ones. Assert a relevant item is in top-3. - const top3 = res.results.slice(0, 3).map((m) => m.content); - expect(top3.some((c) => c.includes("fox") || c.includes("dog"))).toBe( - true, - ); - }); - - test("5. tree paths reflect ~ (home) and share conventions", async () => { - await meJson(["create", "personal note", "--tree", "~/notes"]); - await meJson(["create", "team note", "--tree", "share/team"]); - - const r = await me(["memory", "tree"]); - expect(r.code).toBe(0); - expect(r.stdout).toContain("notes"); - expect(r.stdout).toContain("team"); - }); - - test("6. update + delete round-trip", async () => { - const created = await meJson<{ id: string }>([ - "create", - "ephemeral memory to edit", - "--tree", - "share", - ]); - const updated = await meJson<{ id: string; content: string }>([ - "memory", - "update", - created.id, - "--content", - "edited content", - ]); - expect(updated.content).toBe("edited content"); - - const del = await me(["memory", "delete", created.id, "--yes"]); - expect(del.code).toBe(0); - - // Getting it now fails with a non-zero exit. - const get = await me(["memory", "get", created.id]); - expect(get.code).not.toBe(0); - }); - - // ------------------------------------------------------------------------- - // Extended scenarios - // ------------------------------------------------------------------------- - - test("7. api-key auth works end-to-end (no session token)", async () => { - // Mint the key through the real CLI: create the agent, add it to the - // space, then mint a key for it. - const agent = await meJson<{ id: string }>([ - "agent", - "create", - `bot-${rand()}`, - ]); - await me(["agent", "add", agent.id]); // bring the agent into the space - // Agents join with no grant (their access is clamped to the owner's), so - // grant read on `share` — where the fox memory lives — to make it readable. - await meJson(["access", "grant", agent.id, "share", "r"]); - const key = await meJson<{ id: string; key: string }>([ - "apikey", - "create", - agent.id, - ]); - expect(key.key).toMatch(/^me\./); - - // Search with ONLY the api key — no session token. The agent's global key - // plus X-Me-Space (ME_SPACE) selects the space; this exercises the CLI's - // api-key auth path against the real server end-to-end. - const res = await meJson<{ total: number }>( - ["search", "--fulltext", "fox"], - { ME_API_KEY: key.key, ME_SESSION_TOKEN: "" }, - ); - expect(res.total).toBeGreaterThan(0); - }); - - test("8. `me claude import` backfills work that predates the hook", async () => { - // The scenario: a user does a bunch of Claude Code work BEFORE installing - // the capture hook (no hook fires for it), then installs the hook (which - // begins capturing new sessions live), then runs `me claude import`. The - // pre-install work must be backfilled — the importer has no lower time - // bound tied to hook install; it sweeps every transcript and dedupes by - // deterministic message id. - const root = await mkdtemp(join(tmpdir(), "me-e2e-backfill-")); - const projDir = join(root, "proj"); - await mkdir(projDir, { recursive: true }); - - // cwd "/work/backfill-proj" → no git repo on disk → slug = basename, so - // both sessions land under the same tree. - const cwd = "/work/backfill-proj"; - const tree = "share.projects.backfill_proj.agent_sessions"; - - const mkMsg = - (sessionId: string) => - (i: number, type: "user" | "assistant", text: string) => ({ - type, - uuid: `${sessionId}-${type}-${i}`, - timestamp: `2026-02-01T00:00:0${i}.000Z`, - sessionId, - cwd, - message: - type === "user" - ? { content: text } - : { content: [{ type: "text", text }], model: "claude-x" }, - }); - - const writeTranscript = async (sessionId: string, prefix: string) => { - const m = mkMsg(sessionId); - const lines = [ - m(0, "user", `${prefix} first question`), - m(1, "assistant", `${prefix} first answer`), - m(2, "user", `${prefix} second question`), - m(3, "assistant", `${prefix} second answer`), - ]; - const path = join(projDir, `${sessionId}.jsonl`); - await writeFile(path, lines.map((l) => JSON.stringify(l)).join("\n")); - return path; - }; - - // 1. Pre-install work: a transcript sits on disk; NO hook ever fires for - // it. It must not be in the engine yet. - const oldSession = `pre-install-${rand()}`; - await writeTranscript(oldSession, "old"); - expect(await countBySession(oldSession)).toBe(0); - - // 2. Install the hook (it now captures live) and let it import a NEW - // session — the real `me claude hook` path, reading from stdin. - const newSession = `post-install-${rand()}`; - const newTranscript = await writeTranscript(newSession, "new"); - const hook = await meStdin( - ["claude", "hook", "--event", "stop"], - JSON.stringify({ - transcript_path: newTranscript, - session_id: newSession, - }), - ); - expect(hook.code, hook.stderr).toBe(0); - // The hook captured only the post-install session — the old one is still - // absent (this is exactly the gap `me claude import` must close). - expect(await countBySession(newSession)).toBe(4); - expect(await countBySession(oldSession)).toBe(0); - - // 3. Run the import (canonical spelling; test 9 covers the - // `me claude import` alias). - const imp = await me(["import", "claude", "--source", root]); - expect(imp.code, imp.stderr).toBe(0); - - // 4. The pre-install work is now backfilled, and the hook's live capture - // was not duplicated. - expect(await countBySession(oldSession)).toBe(4); - expect(await countBySession(newSession)).toBe(4); - expect(await countUnder(tree)).toBe(8); - - await rm(root, { recursive: true, force: true }); + return { + ...env, + HOME: tmpHome, + XDG_CONFIG_HOME: join(tmpHome, ".config"), + ME_NO_KEYCHAIN: "1", + ME_SERVER: srv.url, + ME_SESSION_TOKEN: token, + ME_SPACE: spaceSlug, + ...extra, + }; + } + + async function me( + args: string[], + extraEnv?: Record, + cwd?: string, + ): Promise<{ stdout: string; stderr: string; code: number }> { + const proc = Bun.spawn([process.execPath, CLI, ...args], { + env: cliEnv(extraEnv), + stdout: "pipe", + stderr: "pipe", + ...(cwd ? { cwd } : {}), }); - - test("8b. `me claude init` backfills sessions and writes a CLAUDE.md pointer", async () => { - // `me claude init` is the one-shot setup command. Two steps exercised - // here: - // 1. import THIS project's existing sessions (sessions whose recorded - // cwd is at/under init's cwd — init is per-project setup; the - // machine-wide sweep is `me import claude`); - // 2. record the project's memory location in the project's CLAUDE.md - // (the project = init's cwd; not a git repo here → CLAUDE.md lands in - // that dir, slug = its basename). - // The project we run `init` in — a non-git temp dir with a known basename - // so the derived slug is predictable. CLAUDE.md will be written here, and - // the transcript's session records this dir as its cwd, so the session - // tree and the CLAUDE.md pointer name the same project. - const projectRoot = await mkdtemp(join(tmpdir(), "me-e2e-initcwd-")); - const projectDir = join(projectRoot, "initcwd"); - await mkdir(projectDir, { recursive: true }); - // The recorded session cwd must be the REAL path (as Claude Code would - // record it): macOS tmpdir is a symlink (/var/folders → /private/var), - // and init filters against the resolved process.cwd(). - const projectCwd = await realpath(projectDir); - - const sessionId = `init-${rand()}`; - const foreignId = `foreign-${rand()}`; - const tree = "share.projects.initcwd.agent_sessions"; - const mkMsg = ( - sid: string, - cwd: string, - i: number, - type: "user" | "assistant", - text: string, - ) => ({ - type, - uuid: `${sid}-${type}-${i}`, - timestamp: `2026-03-01T00:00:0${i}.000Z`, - sessionId: sid, - cwd, - message: - type === "user" - ? { content: text } - : { content: [{ type: "text", text }], model: "claude-x" }, - }); - // Mirror Claude Code's on-disk layout: each transcript lives in a - // directory named after the session cwd (encoded) — the scoped import - // prunes by that name, so a literal fixture dir would never be scanned. - const writeTranscript = async ( - sid: string, - cwd: string, - prefix: string, - ) => { - const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); - await mkdir(dir, { recursive: true }); - await writeFile( - join(dir, `${sid}.jsonl`), - [ - mkMsg(sid, cwd, 0, "user", `${prefix} first question`), - mkMsg(sid, cwd, 1, "assistant", `${prefix} first answer`), - mkMsg(sid, cwd, 2, "user", `${prefix} second question`), - mkMsg(sid, cwd, 3, "assistant", `${prefix} second answer`), - ] - .map((l) => JSON.stringify(l)) - .join("\n"), - ); - }; - await writeTranscript(sessionId, projectCwd, "init"); - // A session from a DIFFERENT project must not be swept up by init. - await writeTranscript(foreignId, "/work/other-proj", "foreign"); - - // Pre-init: nothing captured, no CLAUDE.md. - expect(await countBySession(sessionId)).toBe(0); - - // Run `init` FROM the project dir so its cwd → slug → CLAUDE.md location. - const init = await me( - ["claude", "init", "--skip-plugin-install"], - undefined, - projectDir, - ); - expect(init.code, init.stderr).toBe(0); - - // Step 1: this project's session was backfilled; the foreign one wasn't. - expect(await countBySession(sessionId)).toBe(4); - expect(await countUnder(tree)).toBe(4); - expect(await countBySession(foreignId)).toBe(0); - - // Step 2: CLAUDE.md now points at this project's memories. - const claudeMd = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); - expect(claudeMd).toContain("memory-engine:start"); - expect(claudeMd).toContain("share.projects.initcwd"); - expect(claudeMd).toContain("share.projects.initcwd.agent_sessions"); - expect(claudeMd).toContain("share.projects.initcwd.git_history"); - - // Re-running is idempotent: still exactly one managed block. - const init2 = await me( - ["claude", "init", "--skip-plugin-install"], - undefined, - projectDir, - ); - expect(init2.code, init2.stderr).toBe(0); - const claudeMd2 = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); - expect(claudeMd2.split("memory-engine:start").length - 1).toBe(1); - - for (const cwd of [projectCwd, "/work/other-proj"]) { - await rm(join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)), { - recursive: true, - force: true, - }); - } - await rm(projectRoot, { recursive: true, force: true }); - }); - - test("8c. `me claude init` honors --skip-transcript-import / --skip-claude-md", async () => { - // Non-interactive (piped) init runs every step except those turned off by - // a --skip- flag. Verify each flag suppresses exactly its step. - // Each case gets its own project dir + a transcript recorded IN that dir - // (init's import is scoped to the project it runs in), stored under the - // Claude Code encoded-cwd directory layout the scoped import prunes by. - const transcriptDirs: string[] = []; - const mkProject = async (name: string) => { - const root = await mkdtemp(join(tmpdir(), "me-e2e-skip-")); - const dir = join(root, name); - await mkdir(dir, { recursive: true }); - return { root, dir }; - }; - const writeTranscript = async (sid: string, cwd: string) => { - const mkMsg = (i: number, type: "user" | "assistant") => ({ - type, - uuid: `${sid}-${type}-${i}`, - timestamp: `2026-04-01T00:00:0${i}.000Z`, - sessionId: sid, - cwd, - message: - type === "user" - ? { content: `q${i}` } - : { - content: [{ type: "text", text: `a${i}` }], - model: "claude-x", - }, - }); - const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); - transcriptDirs.push(dir); - await mkdir(dir, { recursive: true }); - await writeFile( - join(dir, `${sid}.jsonl`), - [ - mkMsg(0, "user"), - mkMsg(1, "assistant"), - mkMsg(2, "user"), - mkMsg(3, "assistant"), - ] - .map((l) => JSON.stringify(l)) - .join("\n"), - ); - }; - - // --skip-transcript-import: CLAUDE.md is written, but this project's - // session is NOT imported (it would have been without the flag). - const a = await mkProject("skipimport"); - const sessionA = `skipa-${rand()}`; - await writeTranscript(sessionA, await realpath(a.dir)); - const r1 = await me( - ["claude", "init", "--skip-transcript-import", "--skip-plugin-install"], - undefined, - a.dir, - ); - expect(r1.code, r1.stderr).toBe(0); - expect(await countBySession(sessionA)).toBe(0); - expect(existsSync(join(a.dir, "CLAUDE.md"))).toBe(true); - - // --skip-claude-md: the project's session imports, but no CLAUDE.md. - const b = await mkProject("skipclaudemd"); - const sessionB = `skipb-${rand()}`; - await writeTranscript(sessionB, await realpath(b.dir)); - const r2 = await me( - ["claude", "init", "--skip-claude-md", "--skip-plugin-install"], - undefined, - b.dir, - ); - expect(r2.code, r2.stderr).toBe(0); - expect(await countBySession(sessionB)).toBe(4); - expect(existsSync(join(b.dir, "CLAUDE.md"))).toBe(false); - - for (const dir of transcriptDirs) { - await rm(dir, { recursive: true, force: true }); - } - await rm(a.root, { recursive: true, force: true }); - await rm(b.root, { recursive: true, force: true }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const code = await proc.exited; + return { stdout, stderr, code }; + } + + // Like `me`, but pipes `input` to the process's stdin (for `me claude hook`, + // which reads the event JSON from stdin). + async function meStdin( + args: string[], + input: string, + extraEnv?: Record, + ): Promise<{ stdout: string; stderr: string; code: number }> { + const proc = Bun.spawn([process.execPath, CLI, ...args], { + env: cliEnv(extraEnv), + stdin: new TextEncoder().encode(input), + stdout: "pipe", + stderr: "pipe", }); - - // Run git in `dir`, isolated from the developer's git config (gpg - // signing, hooks, templates), with deterministic commit dates. - async function git( - dir: string, - args: string[], - dateIso?: string, - ): Promise { - const proc = Bun.spawn( - [ - "git", - "-C", - dir, - "-c", - "user.name=E2E", - "-c", - "user.email=e2e@example.test", - "-c", - "commit.gpgsign=false", - ...args, - ], - { - env: { - ...process.env, - GIT_CONFIG_GLOBAL: "/dev/null", - GIT_CONFIG_SYSTEM: "/dev/null", - ...(dateIso - ? { GIT_AUTHOR_DATE: dateIso, GIT_COMMITTER_DATE: dateIso } - : {}), - }, - stdout: "pipe", - stderr: "pipe", - }, - ); - const code = await proc.exited; - if (code !== 0) { - const stderr = await new Response(proc.stderr).text(); - throw new Error(`git ${args.join(" ")} failed: ${stderr}`); - } - } - - test("8d. `me import git` imports commit history, idempotently and incrementally", async () => { - // A real repo with a known-basename root so the slug (no remote → - // basename) and therefore the tree are predictable. - const root = await mkdtemp(join(tmpdir(), "me-e2e-git-")); - const name = `gitproj${rand()}`; - const repo = join(root, name); - await mkdir(repo, { recursive: true }); - const tree = `share.projects.${name}.git_history`; - - await git(repo, ["init", "-q", "-b", "main"]); - const commitFile = async (file: string, msg: string, dateIso: string) => { - await writeFile(join(repo, file), `${msg}\n`); - await git(repo, ["add", "."], dateIso); - await git(repo, ["commit", "-q", "-m", msg], dateIso); - }; - await commitFile("a.txt", "feat: add a", "2026-05-01T10:00:00Z"); - await commitFile("b.txt", "fix: adjust b", "2026-05-02T10:00:00Z"); - await commitFile("c.txt", "docs: describe c", "2026-05-03T10:00:00Z"); - - // 1. First import: all three commits land under the project tree. - const first = await meJson<{ - inserted: number; - commitsWalked: number; - tree: string; - }>(["import", "git", repo]); - expect(first.tree).toBe(tree); - expect(first.commitsWalked).toBe(3); - expect(first.inserted).toBe(3); - expect(await countUnder(tree)).toBe(3); - - // Spot-check one record's shape: type/sha meta + commit-date temporal + - // file list in the content. + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const code = await proc.exited; + return { stdout, stderr, code }; + } + + // Count memories under a tree in this run's space schema. + async function countUnder(treePrefix: string): Promise { + const [row] = await sql.unsafe( + `select count(*)::int as n from metest_${spaceSlug}.memory + where tree <@ $1::ltree`, + [treePrefix], + ); + return (row?.n as number) ?? 0; + } + + // Count memories captured from a given source session id. + async function countBySession(sessionId: string): Promise { + const [row] = await sql.unsafe( + `select count(*)::int as n from metest_${spaceSlug}.memory + where meta->>'source_session_id' = $1`, + [sessionId], + ); + return (row?.n as number) ?? 0; + } + + // Parse the --json stdout of a `me` invocation, asserting success. + async function meJson( + args: string[], + extraEnv?: Record, + ): Promise { + const r = await me([...args, "--json"], extraEnv); + expect( + r.code, + `expected exit 0 for \`me ${args.join(" ")}\`\nstdout: ${r.stdout}\nstderr: ${r.stderr}`, + ).toBe(0); + return JSON.parse(r.stdout) as T; + } + + // Poll the space schema until N memories have a non-null embedding. + async function waitForEmbeddings(count: number, timeoutMs = 30000) { + const schema = `metest_${spaceSlug}`; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { const [row] = await sql.unsafe( - `select content, meta from metest_${spaceSlug}.memory - where tree = $1::ltree and content like 'fix: adjust b%'`, - [tree], - ); - expect(row?.meta?.type).toBe("git_commit"); - expect(row?.meta?.sha).toMatch(/^[0-9a-f]{40}$/); - expect(row?.meta?.author_email).toBe("e2e@example.test"); - expect(row?.content).toContain("Files:"); - expect(row?.content).toContain("b.txt (+1 -0)"); - - // 2. Plain re-run: the high-water commit is HEAD → incremental walk of - // an empty range; nothing re-sent, nothing duplicated. - const rerun = await meJson<{ inserted: number; commitsWalked: number }>([ - "import", - "git", - repo, - ]); - expect(rerun.commitsWalked).toBe(0); - expect(rerun.inserted).toBe(0); - expect(await countUnder(tree)).toBe(3); - - // 3. --full re-run: walks everything; deterministic ids make the server - // skip every row (`ON CONFLICT DO NOTHING`). - const full = await meJson<{ - inserted: number; - skipped: number; - commitsWalked: number; - }>(["import", "git", "--full", repo]); - expect(full.commitsWalked).toBe(3); - expect(full.inserted).toBe(0); - expect(full.skipped).toBe(3); - expect(await countUnder(tree)).toBe(3); - - // 4. New work: one regular commit + one body-less merge. The next plain - // run walks only the new range, imports the commit, and drops the - // boilerplate merge. - await git(repo, ["checkout", "-q", "-b", "feat"], undefined); - await commitFile("d.txt", "feat: add d", "2026-05-04T10:00:00Z"); - await git(repo, ["checkout", "-q", "main"]); - await git( - repo, - ["merge", "-q", "--no-ff", "feat", "-m", "Merge branch 'feat'"], - "2026-05-05T10:00:00Z", - ); - const incr = await meJson<{ - inserted: number; - commitsWalked: number; - skippedMerges: number; - range?: string; - }>(["import", "git", repo]); - expect(incr.range).toMatch(/^[0-9a-f]{40}\.\.HEAD$/); - expect(incr.commitsWalked).toBe(2); - expect(incr.inserted).toBe(1); - expect(incr.skippedMerges).toBe(1); - expect(await countUnder(tree)).toBe(4); - - await rm(root, { recursive: true, force: true }); - }); - - test("8e. `me claude init` runs the git step; --skip-git-import suppresses it", async () => { - const root = await mkdtemp(join(tmpdir(), "me-e2e-gitinit-")); - const name = `gitinit${rand()}`; - const repo = join(root, name); - await mkdir(repo, { recursive: true }); - const tree = `share.projects.${name}.git_history`; - - await git(repo, ["init", "-q", "-b", "main"]); - await writeFile(join(repo, "x.txt"), "x\n"); - await git(repo, ["add", "."], "2026-05-01T10:00:00Z"); - await git( - repo, - ["commit", "-q", "-m", "feat: initial"], - "2026-05-01T10:00:00Z", + `select count(*)::int as n from ${schema}.memory where embedding is not null`, ); - - // --skip-git-import: no commit memories. - const skipped = await me( - ["claude", "init", "--skip-git-import", "--skip-plugin-install"], - undefined, - repo, - ); - expect(skipped.code, skipped.stderr).toBe(0); - expect(await countUnder(tree)).toBe(0); - - // Plain init (non-interactive baseline) imports the repo's history and - // the CLAUDE.md pointer names the git_history node. - const init = await me( - ["claude", "init", "--skip-plugin-install"], - undefined, - repo, - ); - expect(init.code, init.stderr).toBe(0); - expect(await countUnder(tree)).toBe(1); - const claudeMd = await readFile(join(repo, "CLAUDE.md"), "utf8"); - expect(claudeMd).toContain(`${tree}\``); - - await rm(root, { recursive: true, force: true }); - }); - - test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { - // A minimal Claude Code session transcript on disk. The importer scans - // //*.jsonl; the hook reads the file directly. - const sessionId = `xact-${rand()}`; - const root = await mkdtemp(join(tmpdir(), "me-e2e-transcript-")); - const projDir = join(root, "proj"); - await mkdir(projDir, { recursive: true }); - const transcript = join(projDir, `${sessionId}.jsonl`); - // Two user turns so the importer doesn't skip it as a trivial session - // (the hook captures regardless; this makes both paths process all four). - const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + if ((row?.n ?? 0) >= count) return; + await Bun.sleep(250); + } + throw new Error(`timed out waiting for ${count} embeddings`); + } + + // ------------------------------------------------------------------------- + // Core scenarios + // ------------------------------------------------------------------------- + + test("1. whoami reports the provisioned identity", async () => { + const r = await me(["whoami"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("e2e@example.test"); + }); + + test("2. create + tree round-trip (share namespace)", async () => { + const created = await meJson<{ id: string; tree?: string }>([ + "create", + "the quick brown fox jumps over the lazy dog", + "--tree", + "share", + ]); + expect(created.id).toBeTruthy(); + + const r = await me(["memory", "tree"]); + expect(r.code).toBe(0); + expect(r.stdout.toLowerCase()).toContain("share"); + }); + + test("3. fulltext (BM25) search finds the memory", async () => { + const res = await meJson<{ + total: number; + results: { id: string; content: string }[]; + }>(["search", "--fulltext", "fox"]); + expect(res.total).toBeGreaterThan(0); + expect(res.results.some((m) => m.content.includes("quick brown fox"))).toBe( + true, + ); + }); + + test("4. semantic search ranks a paraphrase near the top", async () => { + // Seed a few more memories to make ranking meaningful. + const seed = (text: string) => meJson(["create", text, "--tree", "share"]); + await seed("a dog chased a cat across the yard"); + await seed("the stock market fell sharply on Tuesday"); + await seed("photosynthesis converts sunlight into energy"); + + // 4 created so far in `share` (1 from scenario 2 + 3 here). Wait for the + // worker to embed them. + await waitForEmbeddings(4); + + const res = await meJson<{ + results: { id: string; content: string }[]; + }>(["search", "--semantic", "wild canine leaps over a sleepy hound"]); + // Recall-based: the fox/dog memories should surface near the top, not the + // stock-market or photosynthesis ones. Assert a relevant item is in top-3. + const top3 = res.results.slice(0, 3).map((m) => m.content); + expect(top3.some((c) => c.includes("fox") || c.includes("dog"))).toBe(true); + }); + + test("5. tree paths reflect ~ (home) and share conventions", async () => { + await meJson(["create", "personal note", "--tree", "~/notes"]); + await meJson(["create", "team note", "--tree", "share/team"]); + + const r = await me(["memory", "tree"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("notes"); + expect(r.stdout).toContain("team"); + }); + + test("6. update + delete round-trip", async () => { + const created = await meJson<{ id: string }>([ + "create", + "ephemeral memory to edit", + "--tree", + "share", + ]); + const updated = await meJson<{ id: string; content: string }>([ + "memory", + "update", + created.id, + "--content", + "edited content", + ]); + expect(updated.content).toBe("edited content"); + + const del = await me(["memory", "delete", created.id, "--yes"]); + expect(del.code).toBe(0); + + // Getting it now fails with a non-zero exit. + const get = await me(["memory", "get", created.id]); + expect(get.code).not.toBe(0); + }); + + // ------------------------------------------------------------------------- + // Extended scenarios + // ------------------------------------------------------------------------- + + test("7. api-key auth works end-to-end (no session token)", async () => { + // Mint the key through the real CLI: create the agent, add it to the + // space, then mint a key for it. + const agent = await meJson<{ id: string }>([ + "agent", + "create", + `bot-${rand()}`, + ]); + await me(["agent", "add", agent.id]); // bring the agent into the space + // Agents join with no grant (their access is clamped to the owner's), so + // grant read on `share` — where the fox memory lives — to make it readable. + await meJson(["access", "grant", agent.id, "share", "r"]); + const key = await meJson<{ id: string; key: string }>([ + "apikey", + "create", + agent.id, + ]); + expect(key.key).toMatch(/^me\./); + + // Search with ONLY the api key — no session token. The agent's global key + // plus X-Me-Space (ME_SPACE) selects the space; this exercises the CLI's + // api-key auth path against the real server end-to-end. + const res = await meJson<{ total: number }>( + ["search", "--fulltext", "fox"], + { ME_API_KEY: key.key, ME_SESSION_TOKEN: "" }, + ); + expect(res.total).toBeGreaterThan(0); + }); + + test("8. `me claude import` backfills work that predates the hook", async () => { + // The scenario: a user does a bunch of Claude Code work BEFORE installing + // the capture hook (no hook fires for it), then installs the hook (which + // begins capturing new sessions live), then runs `me claude import`. The + // pre-install work must be backfilled — the importer has no lower time + // bound tied to hook install; it sweeps every transcript and dedupes by + // deterministic message id. + const root = await mkdtemp(join(tmpdir(), "me-e2e-backfill-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + + // cwd "/work/backfill-proj" → no git repo on disk → slug = basename, so + // both sessions land under the same tree. + const cwd = "/work/backfill-proj"; + const tree = "share.projects.backfill_proj.agent_sessions"; + + const mkMsg = + (sessionId: string) => + (i: number, type: "user" | "assistant", text: string) => ({ type, uuid: `${sessionId}-${type}-${i}`, timestamp: `2026-02-01T00:00:0${i}.000Z`, sessionId, - cwd: "/work/idempotent-proj", + cwd, message: type === "user" ? { content: text } : { content: [{ type: "text", text }], model: "claude-x" }, }); + + const writeTranscript = async (sessionId: string, prefix: string) => { + const m = mkMsg(sessionId); const lines = [ - mkMsg(0, "user", "first question"), - mkMsg(1, "assistant", "first answer"), - mkMsg(2, "user", "second question"), - mkMsg(3, "assistant", "second answer"), + m(0, "user", `${prefix} first question`), + m(1, "assistant", `${prefix} first answer`), + m(2, "user", `${prefix} second question`), + m(3, "assistant", `${prefix} second answer`), ]; + const path = join(projDir, `${sessionId}.jsonl`); + await writeFile(path, lines.map((l) => JSON.stringify(l)).join("\n")); + return path; + }; + + // 1. Pre-install work: a transcript sits on disk; NO hook ever fires for + // it. It must not be in the engine yet. + const oldSession = `pre-install-${rand()}`; + await writeTranscript(oldSession, "old"); + expect(await countBySession(oldSession)).toBe(0); + + // 2. Install the hook (it now captures live) and let it import a NEW + // session — the real `me claude hook` path, reading from stdin. + const newSession = `post-install-${rand()}`; + const newTranscript = await writeTranscript(newSession, "new"); + const hook = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ + transcript_path: newTranscript, + session_id: newSession, + }), + ); + expect(hook.code, hook.stderr).toBe(0); + // The hook captured only the post-install session — the old one is still + // absent (this is exactly the gap `me claude import` must close). + expect(await countBySession(newSession)).toBe(4); + expect(await countBySession(oldSession)).toBe(0); + + // 3. Run the import (canonical spelling; test 9 covers the + // `me claude import` alias). + const imp = await me(["import", "claude", "--source", root]); + expect(imp.code, imp.stderr).toBe(0); + + // 4. The pre-install work is now backfilled, and the hook's live capture + // was not duplicated. + expect(await countBySession(oldSession)).toBe(4); + expect(await countBySession(newSession)).toBe(4); + expect(await countUnder(tree)).toBe(8); + + await rm(root, { recursive: true, force: true }); + }); + + test("8b. `me claude init` backfills sessions and writes a CLAUDE.md pointer", async () => { + // `me claude init` is the one-shot setup command. Two steps exercised + // here: + // 1. import THIS project's existing sessions (sessions whose recorded + // cwd is at/under init's cwd — init is per-project setup; the + // machine-wide sweep is `me import claude`); + // 2. record the project's memory location in the project's CLAUDE.md + // (the project = init's cwd; not a git repo here → CLAUDE.md lands in + // that dir, slug = its basename). + // The project we run `init` in — a non-git temp dir with a known basename + // so the derived slug is predictable. CLAUDE.md will be written here, and + // the transcript's session records this dir as its cwd, so the session + // tree and the CLAUDE.md pointer name the same project. + const projectRoot = await mkdtemp(join(tmpdir(), "me-e2e-initcwd-")); + const projectDir = join(projectRoot, "initcwd"); + await mkdir(projectDir, { recursive: true }); + // The recorded session cwd must be the REAL path (as Claude Code would + // record it): macOS tmpdir is a symlink (/var/folders → /private/var), + // and init filters against the resolved process.cwd(). + const projectCwd = await realpath(projectDir); + + const sessionId = `init-${rand()}`; + const foreignId = `foreign-${rand()}`; + const tree = "share.projects.initcwd.agent_sessions"; + const mkMsg = ( + sid: string, + cwd: string, + i: number, + type: "user" | "assistant", + text: string, + ) => ({ + type, + uuid: `${sid}-${type}-${i}`, + timestamp: `2026-03-01T00:00:0${i}.000Z`, + sessionId: sid, + cwd, + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + // Mirror Claude Code's on-disk layout: each transcript lives in a + // directory named after the session cwd (encoded) — the scoped import + // prunes by that name, so a literal fixture dir would never be scanned. + const writeTranscript = async ( + sid: string, + cwd: string, + prefix: string, + ) => { + const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); + await mkdir(dir, { recursive: true }); await writeFile( - transcript, - lines.map((l) => JSON.stringify(l)).join("\n"), - ); - - // cwd "/work/idempotent-proj" → no git repo on disk → slug = basename. - const tree = "share.projects.idempotent_proj.agent_sessions"; - - // 1. Live capture via the real hook (reads transcript_path from stdin, - // auths with the session, writes via importTranscriptFile). - const hook = await meStdin( - ["claude", "hook", "--event", "stop"], - JSON.stringify({ transcript_path: transcript, session_id: sessionId }), - ); - expect(hook.code, hook.stderr).toBe(0); - expect(await countUnder(tree)).toBe(4); - - // 2. `me claude import` over the SAME transcript → no new rows (same tree + - // deterministic ids ⇒ the importer dedupes against the hook's writes). - const imp = await me(["claude", "import", "--source", root]); - expect(imp.code, imp.stderr).toBe(0); - expect(await countUnder(tree)).toBe(4); - - // 3. Re-run the hook → still idempotent. - const hook2 = await meStdin( - ["claude", "hook", "--event", "stop"], - JSON.stringify({ transcript_path: transcript, session_id: sessionId }), + join(dir, `${sid}.jsonl`), + [ + mkMsg(sid, cwd, 0, "user", `${prefix} first question`), + mkMsg(sid, cwd, 1, "assistant", `${prefix} first answer`), + mkMsg(sid, cwd, 2, "user", `${prefix} second question`), + mkMsg(sid, cwd, 3, "assistant", `${prefix} second answer`), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), ); - expect(hook2.code, hook2.stderr).toBe(0); - expect(await countUnder(tree)).toBe(4); - - await rm(root, { recursive: true, force: true }); - }); - - test("9b. a stale importer_version is re-rendered in place on re-import", async () => { - // The server's conditional upsert: re-importing a session rewrites any - // row whose stored meta.importer_version differs from the current - // importer's, and skips the rest — no client-side existing-state read. - const sessionId = `stale-${rand()}`; - const root = await mkdtemp(join(tmpdir(), "me-e2e-stale-")); - const projDir = join(root, "proj"); - await mkdir(projDir, { recursive: true }); - const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + }; + await writeTranscript(sessionId, projectCwd, "init"); + // A session from a DIFFERENT project must not be swept up by init. + await writeTranscript(foreignId, "/work/other-proj", "foreign"); + + // Pre-init: nothing captured, no CLAUDE.md. + expect(await countBySession(sessionId)).toBe(0); + + // Run `init` FROM the project dir so its cwd → slug → CLAUDE.md location. + const init = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + projectDir, + ); + expect(init.code, init.stderr).toBe(0); + + // Step 1: this project's session was backfilled; the foreign one wasn't. + expect(await countBySession(sessionId)).toBe(4); + expect(await countUnder(tree)).toBe(4); + expect(await countBySession(foreignId)).toBe(0); + + // Step 2: CLAUDE.md now points at this project's memories. + const claudeMd = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); + expect(claudeMd).toContain("memory-engine:start"); + expect(claudeMd).toContain("share.projects.initcwd"); + expect(claudeMd).toContain("share.projects.initcwd.agent_sessions"); + expect(claudeMd).toContain("share.projects.initcwd.git_history"); + + // Re-running is idempotent: still exactly one managed block. + const init2 = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + projectDir, + ); + expect(init2.code, init2.stderr).toBe(0); + const claudeMd2 = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); + expect(claudeMd2.split("memory-engine:start").length - 1).toBe(1); + + for (const cwd of [projectCwd, "/work/other-proj"]) { + await rm(join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)), { + recursive: true, + force: true, + }); + } + await rm(projectRoot, { recursive: true, force: true }); + }); + + test("8c. `me claude init` honors --skip-transcript-import / --skip-claude-md", async () => { + // Non-interactive (piped) init runs every step except those turned off by + // a --skip- flag. Verify each flag suppresses exactly its step. + // Each case gets its own project dir + a transcript recorded IN that dir + // (init's import is scoped to the project it runs in), stored under the + // Claude Code encoded-cwd directory layout the scoped import prunes by. + const transcriptDirs: string[] = []; + const mkProject = async (name: string) => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-skip-")); + const dir = join(root, name); + await mkdir(dir, { recursive: true }); + return { root, dir }; + }; + const writeTranscript = async (sid: string, cwd: string) => { + const mkMsg = (i: number, type: "user" | "assistant") => ({ type, - uuid: `${sessionId}-${type}-${i}`, - timestamp: `2026-05-01T00:00:0${i}.000Z`, - sessionId, - cwd: "/work/stale-proj", + uuid: `${sid}-${type}-${i}`, + timestamp: `2026-04-01T00:00:0${i}.000Z`, + sessionId: sid, + cwd, message: type === "user" - ? { content: text } - : { content: [{ type: "text", text }], model: "claude-x" }, + ? { content: `q${i}` } + : { + content: [{ type: "text", text: `a${i}` }], + model: "claude-x", + }, }); + const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); + transcriptDirs.push(dir); + await mkdir(dir, { recursive: true }); await writeFile( - join(projDir, `${sessionId}.jsonl`), + join(dir, `${sid}.jsonl`), [ - mkMsg(0, "user", "stale first question"), - mkMsg(1, "assistant", "stale first answer"), - mkMsg(2, "user", "stale second question"), - mkMsg(3, "assistant", "stale second answer"), + mkMsg(0, "user"), + mkMsg(1, "assistant"), + mkMsg(2, "user"), + mkMsg(3, "assistant"), ] .map((l) => JSON.stringify(l)) .join("\n"), ); - - const first = await meJson<{ inserted: number }>([ - "import", - "claude", - "--source", - root, - ]); - expect(first.inserted).toBe(4); - - // Rewind one row to look like an older importer build wrote it. - const [stale] = await sql.unsafe( - `update metest_${spaceSlug}.memory + }; + + // --skip-transcript-import: CLAUDE.md is written, but this project's + // session is NOT imported (it would have been without the flag). + const a = await mkProject("skipimport"); + const sessionA = `skipa-${rand()}`; + await writeTranscript(sessionA, await realpath(a.dir)); + const r1 = await me( + ["claude", "init", "--skip-transcript-import", "--skip-plugin-install"], + undefined, + a.dir, + ); + expect(r1.code, r1.stderr).toBe(0); + expect(await countBySession(sessionA)).toBe(0); + expect(existsSync(join(a.dir, "CLAUDE.md"))).toBe(true); + + // --skip-claude-md: the project's session imports, but no CLAUDE.md. + const b = await mkProject("skipclaudemd"); + const sessionB = `skipb-${rand()}`; + await writeTranscript(sessionB, await realpath(b.dir)); + const r2 = await me( + ["claude", "init", "--skip-claude-md", "--skip-plugin-install"], + undefined, + b.dir, + ); + expect(r2.code, r2.stderr).toBe(0); + expect(await countBySession(sessionB)).toBe(4); + expect(existsSync(join(b.dir, "CLAUDE.md"))).toBe(false); + + for (const dir of transcriptDirs) { + await rm(dir, { recursive: true, force: true }); + } + await rm(a.root, { recursive: true, force: true }); + await rm(b.root, { recursive: true, force: true }); + }); + + // Run git in `dir`, isolated from the developer's git config (gpg + // signing, hooks, templates), with deterministic commit dates. + async function git( + dir: string, + args: string[], + dateIso?: string, + ): Promise { + const proc = Bun.spawn( + [ + "git", + "-C", + dir, + "-c", + "user.name=E2E", + "-c", + "user.email=e2e@example.test", + "-c", + "commit.gpgsign=false", + ...args, + ], + { + env: { + ...process.env, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + ...(dateIso + ? { GIT_AUTHOR_DATE: dateIso, GIT_COMMITTER_DATE: dateIso } + : {}), + }, + stdout: "pipe", + stderr: "pipe", + }, + ); + const code = await proc.exited; + if (code !== 0) { + const stderr = await new Response(proc.stderr).text(); + throw new Error(`git ${args.join(" ")} failed: ${stderr}`); + } + } + + test("8d. `me import git` imports commit history, idempotently and incrementally", async () => { + // A real repo with a known-basename root so the slug (no remote → + // basename) and therefore the tree are predictable. + const root = await mkdtemp(join(tmpdir(), "me-e2e-git-")); + const name = `gitproj${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + const commitFile = async (file: string, msg: string, dateIso: string) => { + await writeFile(join(repo, file), `${msg}\n`); + await git(repo, ["add", "."], dateIso); + await git(repo, ["commit", "-q", "-m", msg], dateIso); + }; + await commitFile("a.txt", "feat: add a", "2026-05-01T10:00:00Z"); + await commitFile("b.txt", "fix: adjust b", "2026-05-02T10:00:00Z"); + await commitFile("c.txt", "docs: describe c", "2026-05-03T10:00:00Z"); + + // 1. First import: all three commits land under the project tree. + const first = await meJson<{ + inserted: number; + commitsWalked: number; + tree: string; + }>(["import", "git", repo]); + expect(first.tree).toBe(tree); + expect(first.commitsWalked).toBe(3); + expect(first.inserted).toBe(3); + expect(await countUnder(tree)).toBe(3); + + // Spot-check one record's shape: type/sha meta + commit-date temporal + + // file list in the content. + const [row] = await sql.unsafe( + `select content, meta from metest_${spaceSlug}.memory + where tree = $1::ltree and content like 'fix: adjust b%'`, + [tree], + ); + expect(row?.meta?.type).toBe("git_commit"); + expect(row?.meta?.sha).toMatch(/^[0-9a-f]{40}$/); + expect(row?.meta?.author_email).toBe("e2e@example.test"); + expect(row?.content).toContain("Files:"); + expect(row?.content).toContain("b.txt (+1 -0)"); + + // 2. Plain re-run: the high-water commit is HEAD → incremental walk of + // an empty range; nothing re-sent, nothing duplicated. + const rerun = await meJson<{ inserted: number; commitsWalked: number }>([ + "import", + "git", + repo, + ]); + expect(rerun.commitsWalked).toBe(0); + expect(rerun.inserted).toBe(0); + expect(await countUnder(tree)).toBe(3); + + // 3. --full re-run: walks everything; deterministic ids make the server + // skip every row (`ON CONFLICT DO NOTHING`). + const full = await meJson<{ + inserted: number; + skipped: number; + commitsWalked: number; + }>(["import", "git", "--full", repo]); + expect(full.commitsWalked).toBe(3); + expect(full.inserted).toBe(0); + expect(full.skipped).toBe(3); + expect(await countUnder(tree)).toBe(3); + + // 4. New work: one regular commit + one body-less merge. The next plain + // run walks only the new range, imports the commit, and drops the + // boilerplate merge. + await git(repo, ["checkout", "-q", "-b", "feat"], undefined); + await commitFile("d.txt", "feat: add d", "2026-05-04T10:00:00Z"); + await git(repo, ["checkout", "-q", "main"]); + await git( + repo, + ["merge", "-q", "--no-ff", "feat", "-m", "Merge branch 'feat'"], + "2026-05-05T10:00:00Z", + ); + const incr = await meJson<{ + inserted: number; + commitsWalked: number; + skippedMerges: number; + range?: string; + }>(["import", "git", repo]); + expect(incr.range).toMatch(/^[0-9a-f]{40}\.\.HEAD$/); + expect(incr.commitsWalked).toBe(2); + expect(incr.inserted).toBe(1); + expect(incr.skippedMerges).toBe(1); + expect(await countUnder(tree)).toBe(4); + + await rm(root, { recursive: true, force: true }); + }); + + test("8e. `me claude init` runs the git step; --skip-git-import suppresses it", async () => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-gitinit-")); + const name = `gitinit${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + await writeFile(join(repo, "x.txt"), "x\n"); + await git(repo, ["add", "."], "2026-05-01T10:00:00Z"); + await git( + repo, + ["commit", "-q", "-m", "feat: initial"], + "2026-05-01T10:00:00Z", + ); + + // --skip-git-import: no commit memories. + const skipped = await me( + ["claude", "init", "--skip-git-import", "--skip-plugin-install"], + undefined, + repo, + ); + expect(skipped.code, skipped.stderr).toBe(0); + expect(await countUnder(tree)).toBe(0); + + // Plain init (non-interactive baseline) imports the repo's history and + // the CLAUDE.md pointer names the git_history node. + const init = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + repo, + ); + expect(init.code, init.stderr).toBe(0); + expect(await countUnder(tree)).toBe(1); + const claudeMd = await readFile(join(repo, "CLAUDE.md"), "utf8"); + expect(claudeMd).toContain(`${tree}\``); + + await rm(root, { recursive: true, force: true }); + }); + + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { + // A minimal Claude Code session transcript on disk. The importer scans + // //*.jsonl; the hook reads the file directly. + const sessionId = `xact-${rand()}`; + const root = await mkdtemp(join(tmpdir(), "me-e2e-transcript-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + const transcript = join(projDir, `${sessionId}.jsonl`); + // Two user turns so the importer doesn't skip it as a trivial session + // (the hook captures regardless; this makes both paths process all four). + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-02-01T00:00:0${i}.000Z`, + sessionId, + cwd: "/work/idempotent-proj", + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + const lines = [ + mkMsg(0, "user", "first question"), + mkMsg(1, "assistant", "first answer"), + mkMsg(2, "user", "second question"), + mkMsg(3, "assistant", "second answer"), + ]; + await writeFile(transcript, lines.map((l) => JSON.stringify(l)).join("\n")); + + // cwd "/work/idempotent-proj" → no git repo on disk → slug = basename. + const tree = "share.projects.idempotent_proj.agent_sessions"; + + // 1. Live capture via the real hook (reads transcript_path from stdin, + // auths with the session, writes via importTranscriptFile). + const hook = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ transcript_path: transcript, session_id: sessionId }), + ); + expect(hook.code, hook.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + // 2. `me claude import` over the SAME transcript → no new rows (same tree + + // deterministic ids ⇒ the importer dedupes against the hook's writes). + const imp = await me(["claude", "import", "--source", root]); + expect(imp.code, imp.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + // 3. Re-run the hook → still idempotent. + const hook2 = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ transcript_path: transcript, session_id: sessionId }), + ); + expect(hook2.code, hook2.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + await rm(root, { recursive: true, force: true }); + }); + + test("9b. a stale importer_version is re-rendered in place on re-import", async () => { + // The server's conditional upsert: re-importing a session rewrites any + // row whose stored meta.importer_version differs from the current + // importer's, and skips the rest — no client-side existing-state read. + const sessionId = `stale-${rand()}`; + const root = await mkdtemp(join(tmpdir(), "me-e2e-stale-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-05-01T00:00:0${i}.000Z`, + sessionId, + cwd: "/work/stale-proj", + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + await writeFile( + join(projDir, `${sessionId}.jsonl`), + [ + mkMsg(0, "user", "stale first question"), + mkMsg(1, "assistant", "stale first answer"), + mkMsg(2, "user", "stale second question"), + mkMsg(3, "assistant", "stale second answer"), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + + const first = await meJson<{ inserted: number }>([ + "import", + "claude", + "--source", + root, + ]); + expect(first.inserted).toBe(4); + + // Rewind one row to look like an older importer build wrote it. + const [stale] = await sql.unsafe( + `update metest_${spaceSlug}.memory set content = 'STALE RENDER', meta = jsonb_set(meta, '{importer_version}', '"0"') where meta->>'source_session_id' = $1 and meta->>'source_message_id' = $2 returning id`, - [sessionId, `${sessionId}-user-0`], - ); - expect(stale?.id).toBeDefined(); - - // Re-import: exactly the stale row is rewritten, the rest skip. - const second = await meJson<{ - inserted: number; - updated: number; - skipped: number; - failed: number; - }>(["import", "claude", "--source", root]); - expect(second.inserted).toBe(0); - expect(second.updated).toBe(1); - expect(second.skipped).toBe(3); - expect(second.failed).toBe(0); - - const [row] = await sql.unsafe( - `select content, meta->>'importer_version' as v + [sessionId, `${sessionId}-user-0`], + ); + expect(stale?.id).toBeDefined(); + + // Re-import: exactly the stale row is rewritten, the rest skip. + const second = await meJson<{ + inserted: number; + updated: number; + skipped: number; + failed: number; + }>(["import", "claude", "--source", root]); + expect(second.inserted).toBe(0); + expect(second.updated).toBe(1); + expect(second.skipped).toBe(3); + expect(second.failed).toBe(0); + + const [row] = await sql.unsafe( + `select content, meta->>'importer_version' as v from metest_${spaceSlug}.memory where id = $1`, - [stale?.id as string], - ); - expect(row?.content).toBe("stale first question"); - expect(row?.v).toBe("1"); - - await rm(root, { recursive: true, force: true }); - }); - - test("9c. `me import` group: no bare default, memories ≡ memory import", async () => { - // Bare `me import` is a group, not the old file-import alias: it prints - // the subcommand list and exits non-zero. - const bare = await me(["import"]); - expect(bare.code).not.toBe(0); - expect(bare.stdout + bare.stderr).toContain("memories"); - - // Old muscle memory `me import ` no longer parses. - const fileArg = await me(["import", "nosuch.md"]); - expect(fileArg.code).not.toBe(0); - expect(fileArg.stderr).toContain("unknown command"); - - // The file importer lives at `me import memories`, with - // `me memory import` as its alias — both write the same records. - const record = (i: number) => - JSON.stringify({ - content: `import group probe ${i}`, - tree: "share.importgroup", - }); - const viaGroup = await meStdin(["import", "memories", "-"], record(1)); - expect(viaGroup.code, viaGroup.stderr).toBe(0); - const viaAlias = await meStdin(["memory", "import", "-"], record(2)); - expect(viaAlias.code, viaAlias.stderr).toBe(0); - expect(await countUnder("share.importgroup")).toBe(2); - }); - - test("10. failure modes: bad space and missing auth exit non-zero", async () => { - const badSpace = await me(["search", "--fulltext", "fox"], { - ME_SPACE: "doesnotexist1", + [stale?.id as string], + ); + expect(row?.content).toBe("stale first question"); + expect(row?.v).toBe("1"); + + await rm(root, { recursive: true, force: true }); + }); + + test("9c. `me import` group: no bare default, memories ≡ memory import", async () => { + // Bare `me import` is a group, not the old file-import alias: it prints + // the subcommand list and exits non-zero. + const bare = await me(["import"]); + expect(bare.code).not.toBe(0); + expect(bare.stdout + bare.stderr).toContain("memories"); + + // Old muscle memory `me import ` no longer parses. + const fileArg = await me(["import", "nosuch.md"]); + expect(fileArg.code).not.toBe(0); + expect(fileArg.stderr).toContain("unknown command"); + + // The file importer lives at `me import memories`, with + // `me memory import` as its alias — both write the same records. + const record = (i: number) => + JSON.stringify({ + content: `import group probe ${i}`, + tree: "share.importgroup", }); - expect(badSpace.code).not.toBe(0); - - const noAuth = await me(["whoami"], { ME_SESSION_TOKEN: "" }); - expect(noAuth.code).not.toBe(0); + const viaGroup = await meStdin(["import", "memories", "-"], record(1)); + expect(viaGroup.code, viaGroup.stderr).toBe(0); + const viaAlias = await meStdin(["memory", "import", "-"], record(2)); + expect(viaAlias.code, viaAlias.stderr).toBe(0); + expect(await countUnder("share.importgroup")).toBe(2); + }); + + test("10. failure modes: bad space and missing auth exit non-zero", async () => { + const badSpace = await me(["search", "--fulltext", "fox"], { + ME_SPACE: "doesnotexist1", }); - }, -); + expect(badSpace.code).not.toBe(0); + + const noAuth = await me(["whoami"], { ME_SESSION_TOKEN: "" }); + expect(noAuth.code).not.toBe(0); + }); +}); diff --git a/package.json b/package.json index 861be99..936f13d 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "scripts": { "build": "./bun run --filter '@memory.build/cli' build", "build:all": "./bun scripts/build-all.ts", - "check": "./bun i --silent && ./bun scripts/bundle-web-assets.ts && ./bun run typecheck && ./bun run lint --write && ./bun run test --only-failures && ./bun run test:e2e", + "check": "./bun i --silent && ./bun scripts/bundle-web-assets.ts && ./bun run typecheck && ./bun run lint --write && ./bun run test:unit", + "check:full": "./bun run check && ./bun run test --only-failures && TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-$(ghost connect testing_me)}\" ./bun run test:e2e", "clean": "rm -rf packages/cli/dist dist", "docs": "./bun --filter @memory.build/docs-site dev", "docs:build": "./bun --filter @memory.build/docs-site build", diff --git a/packages/embedding/generate.test.ts b/packages/embedding/generate.test.ts index c55e4b8..5076afd 100644 --- a/packages/embedding/generate.test.ts +++ b/packages/embedding/generate.test.ts @@ -7,79 +7,6 @@ import { } from "./generate"; import type { EmbeddingConfig } from "./types"; -// ============================================================================= -// Integration Tests (conditional) -// ============================================================================= - -const RUN_INTEGRATION = process.env.RUN_EMBEDDING_INTEGRATION === "1"; -const OLLAMA_URL = process.env.OLLAMA_URL ?? "http://localhost:11434"; - -const ollamaConfig: EmbeddingConfig = { - provider: "ollama", - model: "nomic-embed-text", - dimensions: 768, - baseUrl: OLLAMA_URL, -}; - -describe.skipIf(!RUN_INTEGRATION)("embedding integration (ollama)", () => { - test("generateEmbedding returns correct dimensions", async () => { - const result = await generateEmbedding("test text", ollamaConfig); - - expect(result.embedding).toBeInstanceOf(Array); - expect(result.embedding.length).toBe(768); - expect(typeof result.embedding[0]).toBe("number"); - expect(result.tokens).toBeGreaterThan(0); - }); - - test("generateEmbedding handles long text", async () => { - const longText = "word ".repeat(10000); // Very long text - const configWithTruncation: EmbeddingConfig = { - ...ollamaConfig, - options: { maxTokens: 8000 }, - }; - - const result = await generateEmbedding(longText, configWithTruncation); - expect(result.embedding.length).toBe(768); - expect(result.tokens).toBeGreaterThan(0); - }); - - test("generateEmbeddings returns results for batch", async () => { - const rows = [ - { id: "1", content: "first document" }, - { id: "2", content: "second document" }, - { id: "3", content: "third document" }, - ]; - - const results = await generateEmbeddings(rows, ollamaConfig); - - expect(results.length).toBe(3); - for (const result of results) { - expect(result.embedding.length).toBe(768); - expect(result.error).toBeUndefined(); - } - }); - - test("generateEmbeddings returns empty array for empty input", async () => { - const results = await generateEmbeddings([], ollamaConfig); - expect(results).toEqual([]); - }); - - test("validateConfig succeeds with valid config", async () => { - await expect(validateConfig(ollamaConfig)).resolves.toBeUndefined(); - }); - - test("validateConfig throws on dimension mismatch", async () => { - const badConfig: EmbeddingConfig = { - ...ollamaConfig, - dimensions: 512, // Wrong dimension - }; - - await expect(validateConfig(badConfig)).rejects.toThrow( - "Dimension mismatch", - ); - }); -}); - // ============================================================================= // OpenAI Integration Tests (conditional) // ============================================================================= @@ -92,7 +19,10 @@ const openaiConfig: EmbeddingConfig = { dimensions: 1536, }; -describe.skipIf(!RUN_OPENAI_INTEGRATION)( +// TEST_CI disables conditional skips: in CI this suite always runs (missing +// credentials fail loudly as test errors, never as a silent skip). Locally +// it stays opt-in via RUN_OPENAI_INTEGRATION=1. +describe.skipIf(!process.env.TEST_CI && !RUN_OPENAI_INTEGRATION)( "embedding integration (openai)", () => { test("generateEmbedding returns correct dimensions", async () => { diff --git a/packs/README.md b/packs/README.md index f88ea02..3737990 100644 --- a/packs/README.md +++ b/packs/README.md @@ -124,7 +124,7 @@ Old-version memories are automatically cleaned up. Memories that exist in both v 3. Generate deterministic memory IDs with `./bun run scripts/pack-uuids.ts ` 4. Validate with `me pack validate packs/your-pack.yaml` 5. Run the cross-pack check: `./bun run scripts/validate-packs.ts` -6. Run the full repo check: `./bun run check` +6. Run the full repo check: `./bun run check:full` 7. Open a pull request All packs must: From 2a6424b9edd2e6e7f2bc2e0845d0a436df4041d5 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 14:49:18 +0200 Subject: [PATCH 149/156] docs: verification runs against the local me-postgres container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test verification in CLAUDE.md now points TEST_DATABASE_URL at the local Docker Postgres (same image CI builds): the full suite drops ~4min→10s and e2e ~65s→18s — WAN round trips to ghost dominate these statement-chatty suites. The ghost testing_me instructions stay, scoped to explicit ghost testing only; the scripts' ghost fallback (when TEST_DATABASE_URL is unset) is unchanged. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 64 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 36bab2b..7752ed6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,25 +86,28 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): # fast, for iterating on one file: ./bun test packages/cli/mcp/install.test.ts -# Full suite (unit + integration). `test` defaults TEST_DATABASE_URL to the -# ghost instance and runs files in parallel (--parallel=2) with a 30s timeout; -# set TEST_DATABASE_URL (e.g. a local Postgres) to override. -./bun run test +# Full suite (unit + integration) against the LOCAL Postgres container +# (--parallel=2, 30s timeout). Without TEST_DATABASE_URL the scripts fall +# back to the remote ghost instance — minutes instead of seconds. +TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres ./bun run test # Fast inner loop (typecheck + lint + unit tests; no database, ~15s) ./bun run check -# Everything: check + full suite vs ghost + the e2e suite (~5 min) -./bun run check:full +# Everything: check + full suite + the e2e suite (~30s against local Postgres) +TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres ./bun run check:full ``` -**Important**: After making code changes, run `./bun run check` (fast, no DB). -Before committing, run `./bun run check:full` — the full suite against ghost -plus the e2e suite. CI is the strict gate: it runs every suite with `TEST_CI=1`, -which disables conditional skips — any new `describe.skipIf` gate **must** -include `!process.env.TEST_CI` in its condition (pattern: -`packages/embedding/generate.test.ts`, `e2e/cli.e2e.test.ts`) so CI never -silently skips it. +**Important — verification runs against the local Postgres**: after making +code changes, run `./bun run check` (fast, no DB). Before committing, run +`check:full` with `TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres` +(the `me-postgres` Docker container; if it isn't running: +`docker start me-postgres || ./bun run pg`). Only run against ghost when +explicitly asked to test against ghost. CI is the strict gate: it runs every +suite with `TEST_CI=1`, which disables conditional skips — any new +`describe.skipIf` gate **must** include `!process.env.TEST_CI` in its +condition (pattern: `packages/embedding/generate.test.ts`, +`e2e/cli.e2e.test.ts`) so CI never silently skips it. > `packages/web` and `packages/docs-site` are excluded from the root typecheck > (they have their own); `check`/`check:full` do not cover them. @@ -112,30 +115,35 @@ silently skips it. ### Database integration tests `*.integration.test.ts` files run against a real PostgreSQL 18 with the -required extensions (citext, ltree, pgvector, pg_textsearch), provisioned with -ghost. `./bun run test` and `check` already target ghost by default (the full -suite, `--parallel=2`, 30s timeout — see above). `test:db` is the focused -variant: it first reclaims orphaned test schemas, then runs **every** -`*.integration.test.ts` under `packages/` (the auth/core/space migration suites -plus the engine/server/worker suites), `--parallel=2`, 30s timeout. Point -`TEST_DATABASE_URL` at a ghost database and run: +required extensions (citext, ltree, pgvector, pg_textsearch). For verification +use the **local `me-postgres` Docker container** (same image CI builds; +`./bun run pg` creates it). `test:db` is the focused variant: it first +reclaims orphaned test schemas, then runs **every** `*.integration.test.ts` +under `packages/` (the auth/core/space migration suites plus the +engine/server/worker suites), `--parallel=2`, 30s timeout: ```bash -TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db +TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres ./bun run test:db ``` -`testing_me` is the dedicated ghost database for these tests. - -To run a single integration file directly, pass `--timeout 30000` (as `test:db` -does). bun's default 5s timeout isn't enough over a remote ghost connection — -the migrating `beforeAll` provisions a full auth/core/space and overruns it, -which surfaces as a misleading "beforeEach/afterEach hook timed out": +A single integration file runs in seconds locally: ```bash -TEST_DATABASE_URL="$(ghost connect testing_me)" \ +TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres \ ./bun test --timeout 30000 packages/database/core/migrate/migrate.integration.test.ts ``` +**Ghost (only when explicitly asked to test against ghost)**: `testing_me` is +the dedicated ghost database; without `TEST_DATABASE_URL` the `test`/`test:db` +scripts fall back to it. Expect minutes instead of seconds (every statement +pays WAN latency), and always pass `--timeout 30000` for single files — bun's +default 5s isn't enough over the remote connection (a migrating `beforeAll` +overruns it, surfacing as a misleading "beforeEach/afterEach hook timed out"): + +```bash +TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db +``` + Isolation is **schema-level** (ghost forbids `create database`): each test provisions its own throwaway schema(s) — `core_test_` for core, `auth_test_` for auth, `metest_` for the space *migration* tests — so From 8e6fc9d130a2af075e2c18b464e5c5115617d961 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 15:06:16 +0200 Subject: [PATCH 150/156] chore: default test scripts to local Postgres; ghost is explicit-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `test`/`check:full` scripts no longer fall back to the remote ghost instance when TEST_DATABASE_URL is unset — a silent fallback to a shared remote database contradicts the local-first verification policy (and for anyone without the ghost CLI it degraded into an empty-string URL and a confusing connection error). They now default to the local me-postgres container, so the bare commands are the fast path; the localhost default also keeps `check:full`'s e2e leg running (its skip gate keys off the env var being set). Ghost remains available by passing TEST_DATABASE_URL="$(ghost connect testing_me)" explicitly; CI overrides with its own container env as before. Bare `./bun run check:full` verified: ~30s, full suite + e2e green against the local container. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 49 +++++++++++++++++++++++-------------------------- DEVELOPMENT.md | 4 ++-- package.json | 4 ++-- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7752ed6..9c37ab0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,27 +86,25 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): # fast, for iterating on one file: ./bun test packages/cli/mcp/install.test.ts -# Full suite (unit + integration) against the LOCAL Postgres container -# (--parallel=2, 30s timeout). Without TEST_DATABASE_URL the scripts fall -# back to the remote ghost instance — minutes instead of seconds. -TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres ./bun run test +# Full suite (unit + integration) — defaults to the LOCAL Postgres container +# at 127.0.0.1:5432 (--parallel=2, 30s timeout); TEST_DATABASE_URL overrides. +./bun run test # Fast inner loop (typecheck + lint + unit tests; no database, ~15s) ./bun run check # Everything: check + full suite + the e2e suite (~30s against local Postgres) -TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres ./bun run check:full +./bun run check:full ``` **Important — verification runs against the local Postgres**: after making code changes, run `./bun run check` (fast, no DB). Before committing, run -`check:full` with `TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres` -(the `me-postgres` Docker container; if it isn't running: -`docker start me-postgres || ./bun run pg`). Only run against ghost when -explicitly asked to test against ghost. CI is the strict gate: it runs every -suite with `TEST_CI=1`, which disables conditional skips — any new -`describe.skipIf` gate **must** include `!process.env.TEST_CI` in its -condition (pattern: `packages/embedding/generate.test.ts`, +`./bun run check:full` — it defaults to the `me-postgres` Docker container +(if it isn't running: `docker start me-postgres || ./bun run pg`). Only run +against ghost when explicitly asked to test against ghost. CI is the strict +gate: it runs every suite with `TEST_CI=1`, which disables conditional skips +— any new `describe.skipIf` gate **must** include `!process.env.TEST_CI` in +its condition (pattern: `packages/embedding/generate.test.ts`, `e2e/cli.e2e.test.ts`) so CI never silently skips it. > `packages/web` and `packages/docs-site` are excluded from the root typecheck @@ -115,30 +113,29 @@ condition (pattern: `packages/embedding/generate.test.ts`, ### Database integration tests `*.integration.test.ts` files run against a real PostgreSQL 18 with the -required extensions (citext, ltree, pgvector, pg_textsearch). For verification -use the **local `me-postgres` Docker container** (same image CI builds; -`./bun run pg` creates it). `test:db` is the focused variant: it first -reclaims orphaned test schemas, then runs **every** `*.integration.test.ts` -under `packages/` (the auth/core/space migration suites plus the -engine/server/worker suites), `--parallel=2`, 30s timeout: +required extensions (citext, ltree, pgvector, pg_textsearch). Everything +defaults to the **local `me-postgres` Docker container** at 127.0.0.1:5432 +(same image CI builds; `./bun run pg` creates it). `test:db` is the focused +variant: it first reclaims orphaned test schemas, then runs **every** +`*.integration.test.ts` under `packages/` (the auth/core/space migration +suites plus the engine/server/worker suites), `--parallel=2`, 30s timeout: ```bash -TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres ./bun run test:db +./bun run test:db ``` A single integration file runs in seconds locally: ```bash -TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:5432/postgres \ - ./bun test --timeout 30000 packages/database/core/migrate/migrate.integration.test.ts +./bun test --timeout 30000 packages/database/core/migrate/migrate.integration.test.ts ``` **Ghost (only when explicitly asked to test against ghost)**: `testing_me` is -the dedicated ghost database; without `TEST_DATABASE_URL` the `test`/`test:db` -scripts fall back to it. Expect minutes instead of seconds (every statement -pays WAN latency), and always pass `--timeout 30000` for single files — bun's -default 5s isn't enough over the remote connection (a migrating `beforeAll` -overruns it, surfacing as a misleading "beforeEach/afterEach hook timed out"): +the dedicated ghost database — point `TEST_DATABASE_URL` at it explicitly. +Expect minutes instead of seconds (every statement pays WAN latency), and +always pass `--timeout 30000` for single files — bun's default 5s isn't +enough over the remote connection (a migrating `beforeAll` overruns it, +surfacing as a misleading "beforeEach/afterEach hook timed out"): ```bash TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f5a1808..6ee1043 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -209,9 +209,9 @@ After login, the server URL is stored as the default in `~/.config/me/credential | `./bun run pg` | Build and start PostgreSQL in Docker | | `./bun run pg:rm` | Stop and remove the PostgreSQL container | | `./bun run psql` | Connect to PostgreSQL with psql | -| `./bun run test` | Run all package tests (unit + integration, vs ghost by default) | +| `./bun run test` | Run all package tests (unit + integration, vs local Postgres by default) | | `./bun run check` | Fast inner loop: typecheck + lint + unit tests (no database) | -| `./bun run check:full` | Everything: check + full suite vs ghost + e2e | +| `./bun run check:full` | Everything: check + full suite + e2e (vs local Postgres by default) | | `./bun run build` | Compile CLI binary (current platform) | | `./bun run build:all` | Cross-compile CLI for all platforms | | `./bun run install:local` | Build and install local CLI binary to your PATH | diff --git a/package.json b/package.json index 936f13d..16a4561 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build": "./bun run --filter '@memory.build/cli' build", "build:all": "./bun scripts/build-all.ts", "check": "./bun i --silent && ./bun scripts/bundle-web-assets.ts && ./bun run typecheck && ./bun run lint --write && ./bun run test:unit", - "check:full": "./bun run check && ./bun run test --only-failures && TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-$(ghost connect testing_me)}\" ./bun run test:e2e", + "check:full": "./bun run check && ./bun run test --only-failures && TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-postgresql://postgres@127.0.0.1:5432/postgres}\" ./bun run test:e2e", "clean": "rm -rf packages/cli/dist dist", "docs": "./bun --filter @memory.build/docs-site dev", "docs:build": "./bun --filter @memory.build/docs-site build", @@ -32,7 +32,7 @@ "release:server": "./bun scripts/release-server.ts", "server": "./bun run packages/server/index.ts", "setup": "./bun scripts/setup.ts", - "test": "TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-$(ghost connect testing_me)}\" ./bun test packages --timeout 30000 --parallel=2", + "test": "TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-postgresql://postgres@127.0.0.1:5432/postgres}\" ./bun test packages --timeout 30000 --parallel=2", "test:db": "./bun run test:db:clean && find packages -name '*.integration.test.ts' -print0 | xargs -0 ./bun test --parallel=2 --timeout 30000", "test:db:clean": "./bun scripts/clean-test-schemas.ts", "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", From 725e4163cb15bd5787fd80ceddfb28d0532457bd Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 15:15:13 +0200 Subject: [PATCH 151/156] docs(readme): quick start is `me login` + `me claude init`; rest moves to Usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The quick start now tells the golden-path story — init at a project root does everything (plugin install, session + git history backfill, CLAUDE.md pointer). The create/search/install commands move to a Usage section, which also surfaces the `me import` group. Two fixes while here: the create example writes to share.* (an arbitrary top-level tree needs a grant most members don't hold), and "Row-Level Security" is replaced with the actual model (tree-scoped grants in SQL; RLS was removed as unperformant). Co-Authored-By: Claude Fable 5 --- README.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4022169..5c6981d 100644 --- a/README.md +++ b/README.md @@ -30,17 +30,35 @@ npm i -g @memory.build/cli # Authenticate me login +# Set up Claude Code memory for a project — run at the project root +cd ~/code/your-project +me claude init +``` + +`me claude init` does the whole setup in one shot: installs the Claude Code +plugin (hooks + slash commands + MCP) if it isn't already, backfills the +project's past Claude Code sessions and git commit history as searchable +memories, and records the project's memory location in `CLAUDE.md` so agents +consult it. From then on, new sessions are captured automatically. + +## Usage + +```bash # Store a memory -me memory create "Auth uses bcrypt with cost 12" --tree design.auth +me memory create "Auth uses bcrypt with cost 12" --tree share.design.auth # Search by meaning + keywords me memory search "how does authentication work" -# Connect to your AI tools +# Import memories, agent sessions, and git history +me import memories notes.md # md / yaml / json / ndjson records +me import claude # all Claude Code sessions on this machine +me import git # a repo's commit history + +# Connect other AI tools (Claude Code is covered by `me claude init`) me opencode install me codex install me gemini install -me claude install # full plugin: hooks + slash commands + MCP ``` ## How it works @@ -52,7 +70,7 @@ Memory Engine runs as an MCP server that AI agents connect to over stdio. Each a - **ltree** for hierarchical tree paths - **JSONB + GIN** for metadata filtering - **tstzrange** for temporal queries -- **Row-Level Security** for access control +- **Tree-scoped access grants** evaluated in the search SQL (no RLS) ## Documentation From ccc8d7633c14407fca5296a344db565240fbdfbb Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 15:51:21 +0200 Subject: [PATCH 152/156] feat(cli): me import git-hook installs a post-commit hook for git history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New `me import git-hook [repo]` installs a marker-delimited managed block into the repo's effective post-commit hook (worktree-aware via `git rev-parse --git-path hooks`): a backgrounded, silenced `me import git` with an absolute invocation, so commits from GUI clients work and the commit never blocks or fails. Because the import is high-water incremental, any fire catches up the entire backlog (pulls, merges, rebases included) — post-commit alone suffices. Re-install replaces the block in place; a foreign hook is preserved and appended to; `--remove` deletes the block (and the file when only the shebang remains). Repos routing hooks through core.hooksPath (husky etc.) are refused with instructions instead of writing into committed files. `me claude init` gains a git-hook step (after git import, `--skip-git-hook`), hidden when not applicable or already installed. Unit tests cover the block upsert/remove helpers; a new e2e test proves the full loop — install, commit, memory appears, --remove. Co-Authored-By: Claude Fable 5 --- README.md | 1 + docs/cli/me-claude.md | 3 +- docs/cli/me-import.md | 49 ++++ e2e/cli.e2e.test.ts | 64 +++++ packages/cli/commands/claude.ts | 13 + packages/cli/commands/import-git-hook.test.ts | 83 ++++++ packages/cli/commands/import-git-hook.ts | 266 ++++++++++++++++++ packages/cli/commands/import-group.ts | 2 + 8 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 packages/cli/commands/import-git-hook.test.ts create mode 100644 packages/cli/commands/import-git-hook.ts diff --git a/README.md b/README.md index 5c6981d..79d1087 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ me memory search "how does authentication work" me import memories notes.md # md / yaml / json / ndjson records me import claude # all Claude Code sessions on this machine me import git # a repo's commit history +me import git-hook # keep it current via a post-commit hook # Connect other AI tools (Claude Code is covered by `me claude init`) me opencode install diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index e4ed4ca..93ed0a8 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -5,7 +5,7 @@ Claude Code integration commands. ## Commands - [me claude install](#me-claude-install) -- install the Memory Engine plugin for Claude Code (full plugin by default, `--mcp-only` for just the MCP server) -- [me claude init](#me-claude-init) -- one-shot setup: backfill sessions, import git history, record the project's memory location in CLAUDE.md +- [me claude init](#me-claude-init) -- one-shot setup: backfill sessions, import git history, install the post-commit hook, record the project's memory location in CLAUDE.md - [me claude hook](#me-claude-hook) -- invoked by the Claude Code plugin to capture events as memories - [me claude import](#me-claude-import) -- import Claude Code sessions from `~/.claude/projects` @@ -68,6 +68,7 @@ Setup is a list of independent steps. In an interactive terminal `init` presents | Install the Claude Code plugin | `--skip-plugin-install` | Runs the same install as [`me claude install`](#me-claude-install) (full plugin, `user` scope, login-session auth). Only offered when the `claude` binary is on PATH and `claude plugin list` doesn't already show the plugin — otherwise the step is hidden entirely. | | Import this project's Claude Code sessions | `--skip-transcript-import` | Backfills sessions recorded in this project (cwd at/under the repo root, temp-dir projects included) from `~/.claude/projects`. For a machine-wide backfill across all projects, run [`me import claude`](me-import.md#me-import-claude--codex--opencode). | | Import git commit history | `--skip-git-import` | Imports the repo's full commit history — the same import as [`me import git`](me-import.md#me-import-git). Skipped automatically when the current directory is not inside a git repo. | +| Install a git post-commit hook | `--skip-git-hook` | Installs the managed hook from [`me import git-hook`](me-import.md#me-import-git-hook) so each commit triggers a background incremental import. Only offered inside a git repo without a `core.hooksPath` manager and when the hook isn't already installed — otherwise the step is hidden entirely. | | Add a memory pointer to CLAUDE.md | `--skip-claude-md` | Upserts a managed block into the project's CLAUDE.md naming the project tree (`share.projects.`), its `agent_sessions` and `git_history` nodes, and how to search them. Idempotent — re-runs replace the block in place. | Re-running `init` is safe: both imports are incremental/idempotent and the CLAUDE.md block is replaced, not duplicated. diff --git a/docs/cli/me-import.md b/docs/cli/me-import.md index 932aca3..f92d60b 100644 --- a/docs/cli/me-import.md +++ b/docs/cli/me-import.md @@ -9,6 +9,7 @@ Get data into Memory Engine — one subcommand per source. - [me import codex](#me-import-claude--codex--opencode) -- import Codex sessions - [me import opencode](#me-import-claude--codex--opencode) -- import OpenCode sessions - [me import git](#me-import-git) -- import a repo's git commit history +- [me import git-hook](#me-import-git-hook) -- install a post-commit hook that keeps git history memories current There is no bare default: `me import ` does not parse — use `me import memories `. @@ -113,3 +114,51 @@ me import git --dry-run -v # preview me import git # full backfill (first run) me import git # later: walks only commits since the last import ``` + +--- + +## me import git-hook + +Install a managed git `post-commit` hook that re-runs [`me import git`](#me-import-git) in the background after every commit, keeping the repo's git history memories current without manual re-runs. + +``` +me import git-hook [repo] +me import git-hook --remove +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `repo` | no | Path inside the repo. Default: the current directory. | + +| Option | Description | +|--------|-------------| +| `--remove` | Remove the managed block (and the hook file, if nothing else remains). | + +### What gets installed + +A marker-delimited managed block in the repo's effective `post-commit` hook (worktree-aware, resolved via `git rev-parse --git-path hooks`): + +```sh +# >>> memory-engine (managed by `me import git-hook`) >>> +# Best-effort and asynchronous: never blocks or fails the commit. +("/path/to/me" import git >/dev/null 2>&1 &) +# <<< memory-engine <<< +``` + +The embedded `me` path is absolute, so commits from GUI git clients (no shell PATH) still trigger the import. If a `post-commit` hook already exists, the block is appended once and the existing script is preserved; re-running `git-hook` replaces the block in place (idempotent, refreshes the embedded path). A foreign hook that exits early never reaches the appended block — move the block up manually in that case. + +Because [`me import git`](#me-import-git) is high-water incremental, **any** hook fire catches up the entire backlog — including commits that arrived via pull, merge, or rebase since the last fire. A single `post-commit` hook therefore suffices; there is no post-merge/post-rewrite matrix to install. + +### Hooks managers (core.hooksPath) + +When the repo routes hooks through `core.hooksPath` (husky, lefthook, and similar committed hooks managers), `git-hook` refuses rather than write into committed files. Add this line to the manager's `post-commit` hook instead: + +```sh +me import git >/dev/null 2>&1 & +``` + +### Scope and failure mode + +The hook lives in `.git/hooks` — per clone, never committed, never pushed. CI checkouts and teammates' clones are unaffected; each clone opts in by running `me import git-hook` itself ([`me claude init`](me-claude.md#me-claude-init) offers it as a setup step). + +The import is deliberately silent and best-effort: it never blocks or fails a commit, which also means auth or connectivity problems won't surface at commit time. If history seems stale, run `me import git` manually to see the error — the next successful fire catches everything up. diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index b484dc9..48ff898 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -26,6 +26,7 @@ import { readFile, realpath, rm, + stat, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -650,6 +651,7 @@ describe.skipIf( dir: string, args: string[], dateIso?: string, + extraEnv?: Record, ): Promise { const proc = Bun.spawn( [ @@ -672,6 +674,9 @@ describe.skipIf( ...(dateIso ? { GIT_AUTHOR_DATE: dateIso, GIT_COMMITTER_DATE: dateIso } : {}), + // Spawned hooks inherit this env — the git-hook test merges + // cliEnv() here so the hook's `me import git` can reach the server. + ...(extraEnv ?? {}), }, stdout: "pipe", stderr: "pipe", @@ -816,6 +821,65 @@ describe.skipIf( await rm(root, { recursive: true, force: true }); }); + test("8f. `me import git-hook` captures new commits via post-commit", async () => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-githook-")); + const name = `githook${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + await writeFile(join(repo, "a.txt"), "a\n"); + await git(repo, ["add", "."], "2026-05-01T10:00:00Z"); + await git( + repo, + ["commit", "-q", "-m", "feat: first"], + "2026-05-01T10:00:00Z", + ); + + // Install: managed block written, executable, embeds the source invocation. + const install = await me(["import", "git-hook", repo]); + expect(install.code, install.stderr).toBe(0); + const hookFile = join(repo, ".git", "hooks", "post-commit"); + const hook = await readFile(hookFile, "utf8"); + expect(hook).toContain(">>> memory-engine"); + expect(hook).toContain("import git"); + expect(hook).toContain(process.execPath); // bun + index.ts invocation + const { mode } = await stat(hookFile); + expect(mode & 0o111).not.toBe(0); + + // Re-install is idempotent: still exactly one managed block. + const again = await me(["import", "git-hook", repo]); + expect(again.code, again.stderr).toBe(0); + const hook2 = await readFile(hookFile, "utf8"); + expect(hook2.split(">>> memory-engine").length - 1).toBe(1); + + // A commit fires the hook; its background incremental import catches up + // the whole history (both commits). The hook child inherits the commit's + // env, so merge cliEnv() in. + await writeFile(join(repo, "b.txt"), "b\n"); + await git(repo, ["add", "."], "2026-05-02T10:00:00Z", cliEnv()); + await git( + repo, + ["commit", "-q", "-m", "feat: second"], + "2026-05-02T10:00:00Z", + cliEnv(), + ); + const deadline = Date.now() + 30000; + while ((await countUnder(tree)) < 2 && Date.now() < deadline) { + await Bun.sleep(250); + } + expect(await countUnder(tree)).toBe(2); + + // --remove deletes the managed block (and here the whole file, since the + // block was its only content). + const removed = await me(["import", "git-hook", "--remove", repo]); + expect(removed.code, removed.stderr).toBe(0); + expect(existsSync(hookFile)).toBe(false); + + await rm(root, { recursive: true, force: true }); + }); + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { // A minimal Claude Code session transcript on disk. The importer scans // //*.jsonl; the hook reads the file directly. diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 5f62cc7..623c951 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -55,6 +55,7 @@ import { import { getOutputFormat } from "../output.ts"; import { createClaudeImportCommand, runAgentImport } from "./import.ts"; import { runGitImport } from "./import-git.ts"; +import { isGitHookInstallable, runGitHookInstall } from "./import-git-hook.ts"; /** GitHub source for `claude plugin marketplace add`. */ const PLUGIN_MARKETPLACE_SOURCE = "timescale/memory-engine"; @@ -632,6 +633,18 @@ const INIT_STEPS: InitStep[] = [ label: "Import git commit history", run: ({ globalOpts }) => runGitImport({ skipIfNotRepo: true }, globalOpts), }, + { + id: "git-hook", + optionKey: "skipGitHook", + skipFlag: "--skip-git-hook", + skipDescription: "do not install the git post-commit capture hook", + label: "Install a git post-commit hook (keeps git history current)", + // Hidden outside a git repo, when a committed hooks manager owns the + // hook path, or when the managed block is already installed. + available: () => isGitHookInstallable(process.cwd()), + run: ({ globalOpts }) => + runGitHookInstall({ skipIfNotRepo: true }, globalOpts), + }, { id: "claude-md", optionKey: "skipClaudeMd", diff --git a/packages/cli/commands/import-git-hook.test.ts b/packages/cli/commands/import-git-hook.test.ts new file mode 100644 index 0000000..32810a8 --- /dev/null +++ b/packages/cli/commands/import-git-hook.test.ts @@ -0,0 +1,83 @@ +/** + * Tests for the managed post-commit hook block helpers. + */ +import { describe, expect, test } from "bun:test"; +import { + buildHookBlock, + removeHookBlock, + upsertHookScript, +} from "./import-git-hook.ts"; + +const BLOCK = buildHookBlock('"/usr/local/bin/me"'); +const START = "# >>> memory-engine"; + +describe("buildHookBlock", () => { + test("embeds the invocation, backgrounded and silenced", () => { + expect(BLOCK).toContain( + '("/usr/local/bin/me" import git >/dev/null 2>&1 &)', + ); + expect(BLOCK.startsWith(START)).toBe(true); + expect(BLOCK.endsWith("\n")).toBe(true); + }); + + test("supports a two-part source invocation", () => { + const block = buildHookBlock('"/opt/bun" "/repo/packages/cli/index.ts"'); + expect(block).toContain( + '("/opt/bun" "/repo/packages/cli/index.ts" import git >/dev/null 2>&1 &)', + ); + }); +}); + +describe("upsertHookScript", () => { + test("creates a fresh script with a shebang", () => { + const script = upsertHookScript(null, BLOCK); + expect(script.startsWith("#!/bin/sh\n")).toBe(true); + expect(script.split(START).length - 1).toBe(1); + }); + + test("treats an empty file as fresh", () => { + expect(upsertHookScript(" \n", BLOCK).startsWith("#!/bin/sh\n")).toBe( + true, + ); + }); + + test("appends once to a foreign hook, preserving it", () => { + const foreign = '#!/bin/sh\necho "their hook"\n'; + const script = upsertHookScript(foreign, BLOCK); + expect(script).toContain('echo "their hook"'); + expect(script.indexOf(START)).toBeGreaterThan(script.indexOf("their hook")); + expect(script.split(START).length - 1).toBe(1); + }); + + test("re-install replaces the managed block in place without growth", () => { + const v1 = upsertHookScript("#!/bin/sh\necho before\n", BLOCK); + const newBlock = buildHookBlock('"/new/path/me"'); + const v2 = upsertHookScript(v1, newBlock); + expect(v2.split(START).length - 1).toBe(1); + expect(v2).toContain('"/new/path/me"'); + expect(v2).not.toContain("/usr/local/bin/me"); + expect(v2).toContain("echo before"); + // Idempotent: applying the same block again changes nothing. + expect(upsertHookScript(v2, newBlock)).toBe(v2); + }); +}); + +describe("removeHookBlock", () => { + test("returns null when only the shebang would remain", () => { + const script = upsertHookScript(null, BLOCK); + expect(removeHookBlock(script)).toBeNull(); + }); + + test("preserves foreign content", () => { + const foreign = '#!/bin/sh\necho "their hook"\n'; + const script = upsertHookScript(foreign, BLOCK); + const remaining = removeHookBlock(script); + expect(remaining).toContain('echo "their hook"'); + expect(remaining).not.toContain(START); + }); + + test("is a no-op on a script without the block", () => { + const foreign = '#!/bin/sh\necho "their hook"\n'; + expect(removeHookBlock(foreign)).toBe(foreign); + }); +}); diff --git a/packages/cli/commands/import-git-hook.ts b/packages/cli/commands/import-git-hook.ts new file mode 100644 index 0000000..3d0454e --- /dev/null +++ b/packages/cli/commands/import-git-hook.ts @@ -0,0 +1,266 @@ +/** + * `me import git-hook` — install a managed git post-commit hook that keeps a + * repo's git-history memories current. + * + * The hook runs `me import git` in the background after every commit: + * best-effort, asynchronous, silent — it never blocks or fails a commit, and + * the embedded invocation is absolute so GUI git clients (no shell PATH) + * work. Because the import is high-water incremental, ANY fire catches up the + * entire backlog (including commits that arrived via pull/rebase), so a + * single post-commit hook suffices — no post-merge/post-rewrite matrix. + * + * The hook lives in the repo's effective hooks directory as a + * marker-delimited managed block (created, replaced in place, or appended to + * a foreign hook — the same upsert discipline as the CLAUDE.md pointer). + * Repos using a committed hooks manager (`core.hooksPath`, e.g. husky) are + * refused with instructions instead of writing into committed files. + */ +import { execFile } from "node:child_process"; +import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { basename, isAbsolute, join } from "node:path"; +import { promisify } from "node:util"; +import * as clack from "@clack/prompts"; +import { Command } from "commander"; +import { getOutputFormat, output } from "../output.ts"; +import { handleError } from "../util.ts"; + +const execFileAsync = promisify(execFile); + +/** Markers delimiting the managed block inside the hook script. */ +const HOOK_START = "# >>> memory-engine (managed by `me import git-hook`) >>>"; +const HOOK_END = "# <<< memory-engine <<<"; + +const SHEBANG = "#!/bin/sh"; + +/** Quote a path for /bin/sh (double quotes; escapes embedded `"` and `\`). */ +function shQuote(path: string): string { + return `"${path.replace(/([\\"$`])/g, "\\$1")}"`; +} + +/** + * The absolute invocation embedded into the hook, resolved from how this + * process is running: the compiled `me` binary, a source run (`bun + * packages/cli/index.ts` — dev and tests), or `me` on PATH. + */ +export function resolveMeInvocation(): string { + if (basename(process.execPath) === "me") return shQuote(process.execPath); + const entry = process.argv[1]; + if (entry && /\.(ts|js)$/.test(entry) && isAbsolute(entry)) { + return `${shQuote(process.execPath)} ${shQuote(entry)}`; + } + const onPath = Bun.which("me"); + if (onPath) return shQuote(onPath); + throw new Error( + "Cannot resolve the `me` binary to embed in the hook — install it on PATH first.", + ); +} + +/** The managed block (ends with a newline). */ +export function buildHookBlock(invocation: string): string { + return [ + HOOK_START, + "# Best-effort and asynchronous: never blocks or fails the commit.", + `(${invocation} import git >/dev/null 2>&1 &)`, + HOOK_END, + "", + ].join("\n"); +} + +/** + * Upsert the managed block into an existing hook script (null = no file). + * Fresh file → shebang + block; markers present → replaced in place; + * foreign hook → block appended (a foreign script that exits early never + * reaches it — documented limitation). + */ +export function upsertHookScript( + existing: string | null, + block: string, +): string { + if (existing === null || existing.trim().length === 0) { + return `${SHEBANG}\n${block}`; + } + const start = existing.indexOf(HOOK_START); + if (start !== -1) { + const endMarker = existing.indexOf(HOOK_END, start); + const end = + endMarker === -1 ? existing.length : endMarker + HOOK_END.length; + // Swallow a single trailing newline so re-installs don't grow the file. + const tail = existing[end] === "\n" ? end + 1 : end; + return existing.slice(0, start) + block + existing.slice(tail); + } + const sep = existing.endsWith("\n") ? "\n" : "\n\n"; + return existing + sep + block; +} + +/** + * Remove the managed block. Returns the remaining script, or null when + * nothing but the shebang would remain (caller deletes the file). + */ +export function removeHookBlock(existing: string): string | null { + const start = existing.indexOf(HOOK_START); + if (start === -1) return existing; + const endMarker = existing.indexOf(HOOK_END, start); + const end = endMarker === -1 ? existing.length : endMarker + HOOK_END.length; + const tail = existing[end] === "\n" ? end + 1 : end; + const remaining = existing.slice(0, start) + existing.slice(tail); + const meaningful = remaining + .split("\n") + .filter((l) => l.trim().length > 0 && l.trim() !== SHEBANG); + return meaningful.length === 0 ? null : remaining; +} + +async function git(repo: string, args: string[]): Promise { + try { + const { stdout } = await execFileAsync("git", ["-C", repo, ...args], { + timeout: 5000, + encoding: "utf8", + }); + return stdout.trim(); + } catch { + return null; + } +} + +/** + * Whether the git-hook init step should be offered for `cwd`: inside a git + * repo, no committed hooks manager, and the managed block not yet installed. + */ +export async function isGitHookInstallable(cwd: string): Promise { + const root = await git(cwd, ["rev-parse", "--show-toplevel"]); + if (!root) return false; + if (await git(root, ["config", "core.hooksPath"])) return false; + const hooksFile = await resolveHooksFile(root); + try { + const existing = await readFile(hooksFile, "utf8"); + return !existing.includes(HOOK_START); + } catch { + return true; // no hook file yet + } +} + +/** The effective post-commit path (worktree-aware via --git-path). */ +async function resolveHooksFile(root: string): Promise { + const hooksDir = + (await git(root, ["rev-parse", "--git-path", "hooks"])) ?? ""; + const abs = isAbsolute(hooksDir) ? hooksDir : join(root, hooksDir); + return join(abs, "post-commit"); +} + +/** Options for one install/remove run. */ +export interface GitHookOptions { + repo?: string; + remove?: boolean; + /** Soft-skip when the target isn't a git repo (used by `me claude init`). */ + skipIfNotRepo?: boolean; +} + +/** + * Install (or remove) the managed post-commit hook. Exported so + * `me claude init` can run it as a setup step. Purely local — no server + * auth required. + */ +export async function runGitHookInstall( + opts: GitHookOptions, + globalOpts: Record, +): Promise { + const fmt = getOutputFormat(globalOpts); + const repoPath = opts.repo ?? process.cwd(); + + const root = await git(repoPath, ["rev-parse", "--show-toplevel"]); + if (!root) { + if (opts.skipIfNotRepo) { + if (fmt === "text") { + clack.log.info( + `${repoPath} is not a git repository — skipping git hook install`, + ); + } + return; + } + handleError(new Error(`${repoPath} is not a git repository`), fmt); + } + + const hooksFile = await resolveHooksFile(root); + + if (opts.remove) { + let existing: string; + try { + existing = await readFile(hooksFile, "utf8"); + } catch { + output({ hooksFile, action: "absent" }, fmt, () => + clack.log.info(`No hook installed at ${hooksFile}`), + ); + return; + } + const remaining = removeHookBlock(existing); + if (remaining === existing) { + output({ hooksFile, action: "absent" }, fmt, () => + clack.log.info(`No managed block found in ${hooksFile}`), + ); + return; + } + if (remaining === null) await rm(hooksFile); + else await writeFile(hooksFile, remaining); + output({ hooksFile, action: "removed" }, fmt, () => + clack.log.success(`Removed the memory-engine hook from ${hooksFile}`), + ); + return; + } + + // A committed hooks manager (husky, lefthook, …) owns the hook path — + // don't write into committed files; tell the user what to add instead. + const hooksPath = await git(root, ["config", "core.hooksPath"]); + if (hooksPath) { + handleError( + new Error( + `This repo routes hooks through core.hooksPath (${hooksPath}) — a committed hooks manager likely owns it.\n` + + `Add this line to its post-commit hook instead:\n` + + ` me import git >/dev/null 2>&1 &`, + ), + fmt, + ); + } + + let existing: string | null = null; + try { + existing = await readFile(hooksFile, "utf8"); + } catch { + // no hook yet + } + const updated = existing !== null && existing.includes(HOOK_START); + const next = upsertHookScript( + existing, + buildHookBlock(resolveMeInvocation()), + ); + await mkdir(join(hooksFile, ".."), { recursive: true }); + await writeFile(hooksFile, next); + await chmod(hooksFile, 0o755); + + output({ hooksFile, action: updated ? "updated" : "installed" }, fmt, () => { + clack.log.success( + `${updated ? "Updated" : "Installed"} the post-commit hook at ${hooksFile}`, + ); + console.log( + " Each commit now triggers a background `me import git` (incremental,", + ); + console.log( + " silent, never blocks the commit). Remove with: me import git-hook --remove", + ); + }); +} + +/** `me import git-hook` subcommand factory. */ +export function createGitHookCommand(): Command { + return new Command("git-hook") + .description( + "install a git post-commit hook that keeps git history memories current", + ) + .argument("[repo]", "path inside the repo (default: cwd)") + .option("--remove", "remove the managed hook block") + .action(async (repoArg: string | undefined, opts, cmdRef) => { + const globalOpts = cmdRef.optsWithGlobals(); + await runGitHookInstall( + { repo: repoArg, remove: opts.remove === true }, + globalOpts, + ); + }); +} diff --git a/packages/cli/commands/import-group.ts b/packages/cli/commands/import-group.ts index 5842159..c5db9d4 100644 --- a/packages/cli/commands/import-group.ts +++ b/packages/cli/commands/import-group.ts @@ -22,6 +22,7 @@ import { createOpenCodeImportCommand, } from "./import.ts"; import { createGitImportCommand } from "./import-git.ts"; +import { createGitHookCommand } from "./import-git-hook.ts"; import { createMemoryImportCommand } from "./memory-import.ts"; export function createImportCommand(): Command { @@ -33,6 +34,7 @@ export function createImportCommand(): Command { imp.addCommand(createCodexImportCommand("codex")); imp.addCommand(createOpenCodeImportCommand("opencode")); imp.addCommand(createGitImportCommand()); + imp.addCommand(createGitHookCommand()); imp.addHelpText( "after", "\nTo import memory files (the old `me import `), use: me import memories ", From 63f080bbaad948cc9fae6bec1283aa8cef190f6a Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 16:20:57 +0200 Subject: [PATCH 153/156] feat(cli): me claude init shows a checkmark for already-done steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The init availability gate becomes a tri-state: "available" (offer the step), "hidden" (not applicable — no claude binary, not a git repo, a core.hooksPath manager), or "done" (already set up). Done steps print a green ✓ line above the multiselect — "Claude Code plugin already installed" / "Git post-commit hook already installed" — instead of disappearing silently, so users know why the rows are missing. isGitHookInstallable becomes gitHookStatus (installable | not-applicable | installed) to carry the distinction. Co-Authored-By: Claude Fable 5 --- docs/cli/me-claude.md | 4 +- packages/cli/commands/claude.ts | 68 +++++++++++++++++------- packages/cli/commands/import-git-hook.ts | 20 +++---- 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index 93ed0a8..50bfcf2 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -65,10 +65,10 @@ Setup is a list of independent steps. In an interactive terminal `init` presents | Step | Skip flag | What it does | |------|-----------|--------------| -| Install the Claude Code plugin | `--skip-plugin-install` | Runs the same install as [`me claude install`](#me-claude-install) (full plugin, `user` scope, login-session auth). Only offered when the `claude` binary is on PATH and `claude plugin list` doesn't already show the plugin — otherwise the step is hidden entirely. | +| Install the Claude Code plugin | `--skip-plugin-install` | Runs the same install as [`me claude install`](#me-claude-install) (full plugin, `user` scope, login-session auth). Hidden when the `claude` binary isn't on PATH; when `claude plugin list` already shows the plugin, the step is replaced by a ✓ "already installed" line above the picker. | | Import this project's Claude Code sessions | `--skip-transcript-import` | Backfills sessions recorded in this project (cwd at/under the repo root, temp-dir projects included) from `~/.claude/projects`. For a machine-wide backfill across all projects, run [`me import claude`](me-import.md#me-import-claude--codex--opencode). | | Import git commit history | `--skip-git-import` | Imports the repo's full commit history — the same import as [`me import git`](me-import.md#me-import-git). Skipped automatically when the current directory is not inside a git repo. | -| Install a git post-commit hook | `--skip-git-hook` | Installs the managed hook from [`me import git-hook`](me-import.md#me-import-git-hook) so each commit triggers a background incremental import. Only offered inside a git repo without a `core.hooksPath` manager and when the hook isn't already installed — otherwise the step is hidden entirely. | +| Install a git post-commit hook | `--skip-git-hook` | Installs the managed hook from [`me import git-hook`](me-import.md#me-import-git-hook) so each commit triggers a background incremental import. Hidden outside a git repo or when a `core.hooksPath` manager owns the hook path; when the hook is already installed, the step is replaced by a ✓ "already installed" line above the picker. | | Add a memory pointer to CLAUDE.md | `--skip-claude-md` | Upserts a managed block into the project's CLAUDE.md naming the project tree (`share.projects.`), its `agent_sessions` and `git_history` nodes, and how to search them. Idempotent — re-runs replace the block in place. | Re-running `init` is safe: both imports are incremental/idempotent and the CLAUDE.md block is replaced, not duplicated. diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 623c951..4e0f739 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -55,7 +55,7 @@ import { import { getOutputFormat } from "../output.ts"; import { createClaudeImportCommand, runAgentImport } from "./import.ts"; import { runGitImport } from "./import-git.ts"; -import { isGitHookInstallable, runGitHookInstall } from "./import-git-hook.ts"; +import { gitHookStatus, runGitHookInstall } from "./import-git-hook.ts"; /** GitHub source for `claude plugin marketplace add`. */ const PLUGIN_MARKETPLACE_SOURCE = "timescale/memory-engine"; @@ -446,6 +446,10 @@ const CLAUDE_MD_END = ""; const DIM = "\x1b[2m"; const DIM_OFF = "\x1b[22m"; +/** Green checkmark (resets only the foreground color) for already-done init + * steps, matching clack's green log symbols. */ +const CHECK = "\x1b[32m✓\x1b[39m"; + /** * Build the managed CLAUDE.md block that tells an agent where this project's * memories live in Memory Engine and how to search them. `projectTree` is the @@ -540,6 +544,14 @@ interface InitStepContext { server?: string; } +/** + * Availability of an init step in this environment: offer it, hide it + * entirely (not applicable here), or report it as already done — no + * multiselect row, but a ✓ line above the prompt so the user knows it's + * covered. + */ +type StepAvailability = "available" | "hidden" | "done"; + interface InitStep { /** Stable id — the multiselect value and the basis of the --skip flag. */ id: string; @@ -552,11 +564,13 @@ interface InitStep { /** Multiselect row label. */ label: string; /** - * Optional availability gate: a step that resolves false is omitted - * entirely — no multiselect row and not part of the non-interactive - * baseline. Absent means always available. + * Optional availability gate: "hidden" omits the step entirely; "done" + * omits it but prints `doneLabel` with a checkmark. Absent means always + * available. */ - available?: () => Promise; + available?: () => Promise; + /** The ✓ line printed when `available` resolves "done". */ + doneLabel?: string; /** Perform the step. */ run: (ctx: InitStepContext) => Promise; } @@ -578,19 +592,19 @@ export function pluginListShowsInstalled(stdout: string): boolean { } /** - * Availability of the plugin-install init step: offered only when the - * `claude` binary exists and the plugin is not already installed. + * Availability of the plugin-install init step: hidden when the `claude` + * binary is absent, "done" when the plugin is already installed. */ -async function pluginInstallAvailable(): Promise { - if (Bun.which("claude") === null) return false; +async function pluginInstallAvailable(): Promise { + if (Bun.which("claude") === null) return "hidden"; const { exitCode, stdout } = await runCommand([ "claude", "plugin", "list", "--json", ]); - if (exitCode !== 0) return true; // can't tell → offer the install - return !pluginListShowsInstalled(stdout); + if (exitCode !== 0) return "available"; // can't tell → offer the install + return pluginListShowsInstalled(stdout) ? "done" : "available"; } const INIT_STEPS: InitStep[] = [ @@ -600,8 +614,9 @@ const INIT_STEPS: InitStep[] = [ skipFlag: "--skip-plugin-install", skipDescription: "do not install the Claude Code plugin", label: "Install the Claude Code plugin (hooks + slash commands + MCP)", - // Hidden when Claude Code is absent or the plugin is already installed. + // Hidden when Claude Code is absent; ✓ when the plugin is already there. available: pluginInstallAvailable, + doneLabel: "Claude Code plugin already installed", run: ({ server }) => runClaudePluginInstall({ server, scope: "user" }), }, { @@ -639,9 +654,14 @@ const INIT_STEPS: InitStep[] = [ skipFlag: "--skip-git-hook", skipDescription: "do not install the git post-commit capture hook", label: "Install a git post-commit hook (keeps git history current)", - // Hidden outside a git repo, when a committed hooks manager owns the - // hook path, or when the managed block is already installed. - available: () => isGitHookInstallable(process.cwd()), + // Hidden outside a git repo or when a committed hooks manager owns the + // hook path; ✓ when the managed block is already installed. + available: async () => { + const status = await gitHookStatus(process.cwd()); + if (status === "installed") return "done"; + return status === "installable" ? "available" : "hidden"; + }, + doneLabel: "Git post-commit hook already installed", run: ({ globalOpts }) => runGitHookInstall({ skipIfNotRepo: true }, globalOpts), }, @@ -679,13 +699,23 @@ function createClaudeInitCommand(): Command { Boolean(process.stdout.isTTY); // Steps available in this environment (e.g. plugin-install hides itself - // when Claude Code is absent or the plugin is already installed). The - // probe is skipped for steps already opted out non-interactively, so a - // `--skip-` run never pays for that step's availability check. + // when Claude Code is absent). Already-done steps get a ✓ line instead + // of a row, so the user knows they're covered. The probe is skipped for + // steps already opted out non-interactively, so a `--skip-` run + // never pays for that step's availability check. const candidates: InitStep[] = []; for (const step of INIT_STEPS) { if (!interactive && opts[step.optionKey] === true) continue; - if (step.available && !(await step.available())) continue; + const availability = step.available + ? await step.available() + : "available"; + if (availability === "hidden") continue; + if (availability === "done") { + if (fmt === "text") { + clack.log.message(step.doneLabel ?? step.label, { symbol: CHECK }); + } + continue; + } candidates.push(step); } diff --git a/packages/cli/commands/import-git-hook.ts b/packages/cli/commands/import-git-hook.ts index 3d0454e..48cbf92 100644 --- a/packages/cli/commands/import-git-hook.ts +++ b/packages/cli/commands/import-git-hook.ts @@ -121,20 +121,22 @@ async function git(repo: string, args: string[]): Promise { } } -/** - * Whether the git-hook init step should be offered for `cwd`: inside a git - * repo, no committed hooks manager, and the managed block not yet installed. - */ -export async function isGitHookInstallable(cwd: string): Promise { +/** Status of the git hook for `cwd`, driving the `me claude init` step. */ +export type GitHookStatus = + | "installable" // in a repo, no hooks manager, block not yet present + | "not-applicable" // not a git repo, or core.hooksPath owns the hook path + | "installed"; // the managed block is already there + +export async function gitHookStatus(cwd: string): Promise { const root = await git(cwd, ["rev-parse", "--show-toplevel"]); - if (!root) return false; - if (await git(root, ["config", "core.hooksPath"])) return false; + if (!root) return "not-applicable"; + if (await git(root, ["config", "core.hooksPath"])) return "not-applicable"; const hooksFile = await resolveHooksFile(root); try { const existing = await readFile(hooksFile, "utf8"); - return !existing.includes(HOOK_START); + return existing.includes(HOOK_START) ? "installed" : "installable"; } catch { - return true; // no hook file yet + return "installable"; // no hook file yet } } From 2f3c88d3f7f451d1a7173a76d94a172b0924f02f Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 16:29:53 +0200 Subject: [PATCH 154/156] =?UTF-8?q?fix(cli):=20no=20stray=20guide=20line?= =?UTF-8?q?=20between=20consecutive=20init=20=E2=9C=93=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Print the already-done lines as one block: clack's default spacing of 1 puts a bare guide line above each log message, so the second and later ✓ lines now pass spacing 0. Co-Authored-By: Claude Fable 5 --- packages/cli/commands/claude.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 4e0f739..f59872e 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -704,6 +704,7 @@ function createClaudeInitCommand(): Command { // steps already opted out non-interactively, so a `--skip-` run // never pays for that step's availability check. const candidates: InitStep[] = []; + const doneLabels: string[] = []; for (const step of INIT_STEPS) { if (!interactive && opts[step.optionKey] === true) continue; const availability = step.available @@ -711,13 +712,18 @@ function createClaudeInitCommand(): Command { : "available"; if (availability === "hidden") continue; if (availability === "done") { - if (fmt === "text") { - clack.log.message(step.doneLabel ?? step.label, { symbol: CHECK }); - } + doneLabels.push(step.doneLabel ?? step.label); continue; } candidates.push(step); } + if (fmt === "text") { + // One block: spacing 0 keeps consecutive ✓ lines adjacent (clack's + // default spacing of 1 would put a bare guide line between them). + doneLabels.forEach((label, i) => { + clack.log.message(label, { symbol: CHECK, spacing: i === 0 ? 1 : 0 }); + }); + } // Baseline = every available step not turned off via its --skip-* flag. const baseline = candidates.filter((s) => opts[s.optionKey] !== true); From c1aefa946abda811221cf652a015cc9cb34cc3df Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 17:37:31 +0200 Subject: [PATCH 155/156] feat(cli): closing guidance after me claude init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the init steps complete, print a note telling the user what the wiring buys them: Claude now draws on the project's memories (past sessions, git history) automatically for history/architecture questions and while exploring for new features — plus example prompts for invoking memories explicitly. Text mode only; skipped when nothing ran. Co-Authored-By: Claude Fable 5 --- packages/cli/commands/claude.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index f59872e..cfbf234 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -762,10 +762,31 @@ function createClaudeInitCommand(): Command { if (fmt === "text") clack.log.step(step.label); await step.run(ctx); } + if (fmt === "text") printInitOutro(); }); return cmd; } +/** + * Closing guidance after init: what having project memories wired up + * actually buys the user, and how to invoke them deliberately. + */ +function printInitOutro(): void { + clack.note( + [ + "Ask Claude about this project's history or architecture — it now", + "draws on your memories (past sessions, git history) automatically,", + "and consults them when exploring the code for new features.", + "", + "You can also point Claude at them explicitly, e.g.:", + `${DIM}"Search memory: why did we structure the database this way?"${DIM_OFF}`, + `${DIM}"Check memory for past work on this area before we start"${DIM_OFF}`, + `${DIM}"What do my memories say about how deploys work here?"${DIM_OFF}`, + ].join("\n"), + "Your project now has memory", + ); +} + export function createClaudeCommand(): Command { const claude = new Command("claude").description("Claude Code integration"); claude.addCommand(createClaudeInstallCommand()); From 542534286e5d93831b2fb68f208d8c7e4db3add8 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 11 Jun 2026 17:39:04 +0200 Subject: [PATCH 156/156] fix(cli): init outro example prompts name memory engine / me memories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generic "memory" in the example prompts didn't tell Claude which tool to reach for — name the product so the prompts reliably route to the me MCP tools. Co-Authored-By: Claude Fable 5 --- packages/cli/commands/claude.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index cfbf234..e589927 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -779,9 +779,9 @@ function printInitOutro(): void { "and consults them when exploring the code for new features.", "", "You can also point Claude at them explicitly, e.g.:", - `${DIM}"Search memory: why did we structure the database this way?"${DIM_OFF}`, - `${DIM}"Check memory for past work on this area before we start"${DIM_OFF}`, - `${DIM}"What do my memories say about how deploys work here?"${DIM_OFF}`, + `${DIM}"Search memory engine: why did we structure the database this way?"${DIM_OFF}`, + `${DIM}"Check me memories for past work on this area before we start"${DIM_OFF}`, + `${DIM}"What do my me memories say about how deploys work here?"${DIM_OFF}`, ].join("\n"), "Your project now has memory", );